diff --git a/.github/workflows/build-interop-docker.yml b/.github/workflows/build-interop-docker.yml
index bd4a7699b..f8f2eca6a 100644
--- a/.github/workflows/build-interop-docker.yml
+++ b/.github/workflows/build-interop-docker.yml
@@ -5,6 +5,7 @@ on:
       - master
     tags:
       - 'v*'
+  pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -23,6 +24,7 @@ jobs:
         with:
           platforms: linux/amd64,linux/arm64
       - name: Login to Docker Hub
+        if: github.event_name == 'push'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -41,9 +43,8 @@ jobs:
           fi
       - uses: docker/build-push-action@v6
         with:
-          context: "{{defaultContext}}:interop"
+          context: "."
+          file: "interop/Dockerfile"
           platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            GITREF=${{ steps.tag.outputs.gitref }}
+          push: ${{ github.event_name == 'push' }}
           tags: martenseemann/quic-go-interop:${{ steps.tag.outputs.tag }}
diff --git a/interop/Dockerfile b/interop/Dockerfile
index 652a3c1a1..9e819ed29 100644
--- a/interop/Dockerfile
+++ b/interop/Dockerfile
@@ -3,7 +3,7 @@ FROM martenseemann/quic-network-simulator-endpoint:latest AS builder
 ARG TARGETPLATFORM
 RUN echo "TARGETPLATFORM: ${TARGETPLATFORM}"
 
-RUN apt-get update && apt-get install -y wget tar git
+RUN apt-get update && apt-get install -y wget tar git && rm -rf /var/lib/apt/lists/*
 
 ENV GOVERSION=1.24.0
 
@@ -18,13 +18,8 @@ ENV PATH="/go/bin:${PATH}"
 # build with --build-arg CACHEBUST=$(date +%s)
 ARG CACHEBUST=1
 
-# build other branches / commits / tags using --build-arg GITREF="<git reference>"
-ARG GITREF="master"
-
-RUN git clone https://github.com/quic-go/quic-go
+COPY . /quic-go
 WORKDIR /quic-go
-RUN git checkout ${GITREF}
-RUN go get ./...
 
 RUN git rev-parse HEAD | tee commit.txt
 RUN go build -o server -ldflags="-X github.com/quic-go/quic-go/qlog.quicGoVersion=$(git describe --always --long --dirty)" interop/server/main.go
@@ -36,8 +31,8 @@ FROM martenseemann/quic-network-simulator-endpoint:latest
 WORKDIR /quic-go
 
 COPY --from=builder /quic-go/commit.txt /quic-go/server /quic-go/client ./
+COPY --from=builder /quic-go/interop/run_endpoint.sh ./
 
-COPY run_endpoint.sh .
 RUN chmod +x run_endpoint.sh
 
 ENTRYPOINT [ "./run_endpoint.sh" ]