diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eceddfe18bf23c0887c56991ee8d498ba9d3fbb5..756c90a914c9303460ca5da62054bd0823cd7a61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,9 +2,27 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
 
 stages:
+  - test
   - generate-html
   - generate-pdf
 
+test-urls:
+  image: debian:buster-slim
+  stage: test
+  tags:
+    - lightweight
+  before_script:
+  - apt update && apt install -y --no-install-recommends
+      python3
+      python3-markdown
+      python3-requests
+  script:
+  - scripts/test_urls.py content/
+  # We are running this for all commits for all URLs on the website, so it
+  # wouldn't be fair to block a merge due to an unrelated URL suddenly failing.
+  # Treat this as a warning that the site has degraded.
+  allow_failure: true
+
 test-html:
   image:
     name: klakegg/hugo:latest