diff --git a/src/client/InputHandler.ts b/src/client/InputHandler.ts
index 91a719dbe..694e0e493 100644
--- a/src/client/InputHandler.ts
+++ b/src/client/InputHandler.ts
@@ -451,6 +451,10 @@ export class InputHandler {
   }
 
   private onPointerDown(event: PointerEvent) {
+    if (this.uiState.overGhostControls) {
+      return;
+    }
+
     if (event.button === 1) {
       event.preventDefault();
       this.eventBus.emit(new AutoUpgradeEvent(event.clientX, event.clientY));
@@ -478,6 +482,12 @@ export class InputHandler {
   }
 
   onPointerUp(event: PointerEvent) {
+    if (this.uiState.overGhostControls) {
+      this.pointerDown = false;
+      this.pointers.clear();
+      return;
+    }
+
     if (event.button === 1) {
       event.preventDefault();
       return;
diff --git a/src/client/graphics/GameRenderer.ts b/src/client/graphics/GameRenderer.ts
index cd6b1d31e..3263251fd 100644
--- a/src/client/graphics/GameRenderer.ts
+++ b/src/client/graphics/GameRenderer.ts
@@ -55,6 +55,7 @@ export function createRenderer(
     ghostStructure: null,
     rocketDirectionUp: true,
     lockedGhostTile: null,
+    overGhostControls: false,
   } as UIState;
 
   //hide when the game renders
diff --git a/src/client/graphics/UIState.ts b/src/client/graphics/UIState.ts
index 292532466..4350a7b1b 100644
--- a/src/client/graphics/UIState.ts
+++ b/src/client/graphics/UIState.ts
@@ -6,4 +6,5 @@ export interface UIState {
   ghostStructure: UnitType | null;
   rocketDirectionUp: boolean;
   lockedGhostTile: TileRef | null;
+  overGhostControls: boolean;
 }
diff --git a/src/client/graphics/layers/StructureIconsLayer.ts b/src/client/graphics/layers/StructureIconsLayer.ts
index 1e465232d..7d7ae9503 100644
--- a/src/client/graphics/layers/StructureIconsLayer.ts
+++ b/src/client/graphics/layers/StructureIconsLayer.ts
@@ -639,20 +639,25 @@ export class StructureIconsLayer implements Layer {
 
       // Hover effects
       button.on("pointerover", () => {
+        this.uiState.overGhostControls = true;
         button.tint = 0xdddddd;
       });
       button.on("pointerout", () => {
+        this.uiState.overGhostControls = false;
         button.tint = 0xffffff;
       });
       button.on("pointerdown", () => {
+        this.uiState.overGhostControls = true;
         button.tint = 0xaaaaaa;
       });
       button.on("pointerup", () => {
+        this.uiState.overGhostControls = false;
         button.tint = 0xffffff;
       });
 
       button.on("pointertap", (e) => {
         e.stopPropagation();
+        this.uiState.overGhostControls = false;
         onClick();
       });
 
@@ -681,6 +686,7 @@ export class StructureIconsLayer implements Layer {
 
   private destroyGhostControls() {
     if (!this.ghostControls) return;
+    this.uiState.overGhostControls = false;
     this.ghostControls.container.destroy({ children: true });
     this.ghostControls = null;
   }
diff --git a/tests/InputHandler.test.ts b/tests/InputHandler.test.ts
index 27ea31878..779634a73 100644
--- a/tests/InputHandler.test.ts
+++ b/tests/InputHandler.test.ts
@@ -39,6 +39,7 @@ describe("InputHandler AutoUpgrade", () => {
         ghostStructure: null,
         rocketDirectionUp: true,
         lockedGhostTile: null,
+        overGhostControls: false,
       },
       mockCanvas,
       eventBus,