diff --git a/src/ViewerCanvas.tsx b/src/ViewerCanvas.tsx
index 59a0b1f..ed878f3 100644
--- a/src/ViewerCanvas.tsx
+++ b/src/ViewerCanvas.tsx
@@ -56,6 +56,9 @@ export default class ViewerCanvas extends React.Component<ViewerCanvasProps, Vie
   }
 
   handleMouseDown = (e) => {
+    if (e.button !== 0) {
+      return;
+    }
     if (!this.props.visible || !this.props.drag) {
       return;
     }