diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 000000000..4e689bc2d
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,15 @@
+# Git Related Items
+.git
+.github
+.gitignore
+
+# CI Related Items
+.travis.yml
+cloudbuild.yaml
+.golangci.yml
+.zappr.yaml
+
+# Other
+docs
+OWNERS
+vendor
diff --git a/Dockerfile b/Dockerfile
index 8ba67b7b3..8e71170b0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,8 +19,12 @@ ARG ARCH
 
 WORKDIR /sigs.k8s.io/external-dns
 
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
 COPY . .
-RUN make test && make build.$ARCH
+RUN make test build.$ARCH
 
 # final image
 FROM $ARCH/alpine:3.12
diff --git a/Dockerfile.mini b/Dockerfile.mini
index d18516a2d..1f7633006 100644
--- a/Dockerfile.mini
+++ b/Dockerfile.mini
@@ -16,13 +16,17 @@ FROM golang:1.15 as builder
 
 WORKDIR /sigs.k8s.io/external-dns
 
+RUN apt-get update \
+    && apt-get install \
+        ca-certificates \
+    && update-ca-certificates
+
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
 COPY . .
-RUN apt-get update && \
-    apt-get install ca-certificates && \
-    update-ca-certificates && \
-    go mod vendor && \
-    make test && \
-    make build
+RUN make test build
 
 FROM gcr.io/distroless/static