Merge pull request #1456 from aptly-dev/doc/gpg-api

doc: add swagger doc for /api/gpg/key
tests: use faketime for expired keys/signatures

Author: neolynx

.github/workflows/ci.yml

@@ -30,10 +30,10 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: "Install Packages"
+      - name: "Install Test Packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime
 
       - name: "Checkout Repository"
         uses: actions/checkout@v4
@@ -139,7 +139,7 @@ jobs:
         APT_LISTCHANGES_FRONTEND: none
         DEBIAN_FRONTEND: noninteractive
     steps:
-      - name: "Install packages"
+      - name: "Install Build Packages"
         run: |
           apt-get update
           apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \

Makefile

@@ -7,6 +7,9 @@ COVERAGE_DIR?=$(shell mktemp -d)
 GOOS=$(shell go env GOHOSTOS)
 GOARCH=$(shell go env GOHOSTARCH)
 
+# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
+export TEST_FAKETIME := 2025-01-02 03:04:05
+
 # export CAPUTRE=1 for regenrating test gold files
 ifeq ($(CAPTURE),1)
 CAPTURE_ARG := --capture
@@ -86,11 +89,11 @@ install:
 	# go install -v
 	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-test: prepare swagger etcd-install  ## Run unit tests
+test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify which tests to run)
 	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
 	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
 	@echo "\e[33m\e[1mRunning go test ...\e[0m"
-	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
+	faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret
 	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
 	@pid=`cat /tmp/etcd.pid`; kill $$pid
 	@rm -f /tmp/aptly-etcd-data/etcd.log
@@ -104,7 +107,7 @@ system-test: prepare swagger etcd-install  ## Run system tests
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
 	# Run system tests
-	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST)
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST)
 
 bench:
 	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -176,13 +179,13 @@ docker-shell:  ## Run aptly and other commands in docker container
 docker-deb:  ## Build debian packages in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
 
-docker-unit-test:  ## Run unit tests in docker container
+docker-unit-test:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
-		test \
+		test TEST=$(TEST) \
 		azurite-stop
 
 docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)

api/gpg.go

@@ -13,26 +13,34 @@ import (
 )
 
 type gpgAddKeyParams struct {
-	// Keyserver, when downloading GpgKeyIDs
-	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
-	// GpgKeyIDs to download from Keyserver, comma separated list
-	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500,8B48AD6246925553"`
-	// Armored gpg public ket, instead of downloading from keyserver
-	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
 	// Keyring for adding the keys (default: trustedkeys.gpg)
 	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+
+	// Add ASCII armored gpg public key, do not download from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+
+	// Keyserver to download keys provided in `GpgKeyID`
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// Keys do download from `Keyserver`, separated by space
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500 8B48AD6246925553"`
 }
 
 // @Summary Add GPG Keys
 // @Description **Adds GPG keys to aptly keyring**
 // @Description
 // @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Description
+// @Description Keys can be added in two ways:
+// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)
+// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty)
+// @Description
 // @Tags Mirrors
+// @Consume  json
+// @Param request body gpgAddKeyParams true "Parameters"
 // @Produce json
 // @Success 200 {object} string "OK"
 // @Failure 400 {object} Error "Bad Request"
-// @Failure 404 {object} Error "Not Found"
-// @Router /api/gpg [post]
+// @Router /api/gpg/key [post]
 func apiGPGAddKey(c *gin.Context) {
 	b := gpgAddKeyParams{}
 	if c.Bind(&b) != nil {

api/repos.go

@@ -107,9 +107,9 @@ type repoCreateParams struct {
 // @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
 // @Description ```
 // @Tags Repos
-// @Produce  json
 // @Consume  json
 // @Param request body repoCreateParams true "Parameters"
+// @Produce  json
 // @Success 201 {object} deb.LocalRepo
 // @Failure 404 {object} Error "Source snapshot not found"
 // @Failure 409 {object} Error "Local repo already exists"
@@ -178,8 +178,10 @@ type reposEditParams struct {
 // @Summary Update Repository
 // @Description **Update local repository meta information**
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposEditParams true "Parameters"
+// @Produce json
 // @Success 200 {object} deb.LocalRepo "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 500 {object} Error "Internal Server Error"
@@ -231,8 +233,8 @@ func apiReposEdit(c *gin.Context) {
 // @Summary Get Repository Info
 // @Description Returns basic information about local repository.
 // @Tags Repos
-// @Produce  json
 // @Param name path string true "Repository name"
+// @Produce  json
 // @Success 200 {object} deb.LocalRepo
 // @Failure 404 {object} Error "Repository not found"
 // @Router /api/repos/{name} [get]
@@ -254,9 +256,10 @@ func apiReposShow(c *gin.Context) {
 // @Description Cannot drop repos that are published.
 // @Description Needs force=1 to drop repos used as source by other repos.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Param force query int false "force: 1 to enable"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "Repo object"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 404 {object} Error "Repo Conflict"
@@ -306,12 +309,12 @@ func apiReposDrop(c *gin.Context) {
 // @Description ["Pi386 aptly 0.8 966561016b44ed80"]
 // @Description ```
 // @Tags Repos
-// @Produce json
-// @Param name path string true "Snapshot to search"
+// @Param name path string true "Repository name"
 // @Param q query string true "Package query (e.g Name%20(~%20matlab))"
 // @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
 // @Param format query string true "Set to 'details' to return extra info about each package"
 // @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 404 {object} Error "Internal Server Error"
@@ -406,9 +409,10 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 // @Description
 // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
 // @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -426,9 +430,11 @@ func apiReposPackagesAdd(c *gin.Context) {
 // @Description
 // @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
 // @Tags Repos
-// @Produce json
-// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
+// @Consume  json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -608,8 +614,8 @@ type reposCopyPackageParams struct {
 // @Description Copies a package from a source to destination repository
 // @Tags Repos
 // @Produce json
-// @Param name path string true "Source repo"
-// @Param src path string true "Destination repo"
+// @Param name path string true "Destination repo"
+// @Param src path string true "Source repo"
 // @Param file path string true "File/packages to copy"
 // @Param _async query bool false "Run in background and return task object"
 // @Success 200 {object} task.ProcessReturnValue "msg"
@@ -762,12 +768,15 @@ func apiReposCopyPackage(c *gin.Context) {
 // @Summary Include File from Directory
 // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param file path string true "File/packages to include"
 // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
 // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
 // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
 // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/repos/{name}/include/{dir}/{file} [post]
@@ -784,12 +793,14 @@ type reposIncludePackageFromDirResponse struct {
 // @Summary Include Directory
 // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
 // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
 // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
 // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
 // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} reposIncludePackageFromDirResponse "Response"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/repos/{name}/include/{dir} [post]

system/Dockerfile

@@ -6,7 +6,8 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg b
         g++ python3-etcd3 python3-plyvel graphviz devscripts sudo dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu \
         binutils-arm-linux-gnueabihf bash-completion zip ruby-dev lintian npm \
         libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
-        gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu && \
+        gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu \
+        faketime && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m --shell /bin/bash --home-dir /var/lib/aptly aptly

system/lib.py

@@ -130,6 +130,7 @@ class BaseTest(object):
     sortOutput = False
     debugOutput = False
     EtcdServer = None
+    faketime = False
 
     aptlyDir = ".aptly"
     aptlyConfigFile = ".aptly.conf"
@@ -311,6 +312,9 @@ def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None):
             aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test"
             command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]]
 
+        if self.faketime:
+            command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command
+
         environ = os.environ.copy()
         environ["LC_ALL"] = "C"
         environ.update(self.environmentOverride)

system/run.py

@@ -50,6 +50,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     if not coverage_dir:
         coverage_dir = mkdtemp(suffix="aptly-coverage")
 
+    failed = False
     for test in tests:
         orig_stdout = sys.stdout
         orig_stderr = sys.stderr
@@ -157,8 +158,15 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
 
                 t.shutdown()
 
+                if failed:
+                    break
+            if failed:
+                break
+
         sys.stdout = orig_stdout
         sys.stderr = orig_stderr
+        if failed:
+            break
 
     if lastBase is not None:
         lastBase.shutdown_class()

system/s3_lib.py

@@ -24,7 +24,9 @@ class S3Test(BaseTest):
     s3Overrides = {}
 
     def fixture_available(self):
-        return super(S3Test, self).fixture_available() and s3_conn is not None
+        return super(S3Test, self).fixture_available() and \
+                'AWS_SECRET_ACCESS_KEY' in os.environ and 'AWS_ACCESS_KEY_ID' in os.environ and \
+                os.environ['AWS_SECRET_ACCESS_KEY'] != "" and os.environ['AWS_ACCESS_KEY_ID'] != ""
 
     def prepare(self):
         self.bucket_name = "aptly-sys-test-" + str(uuid.uuid1())

system/t04_mirror/create.py

@@ -416,6 +416,7 @@ class CreateMirror31Test(BaseTest):
     runCmd = "aptly mirror create --keyring=aptlytest-gpg1.gpg mirror11 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch"
     configOverride = {"gpgProvider": "internal", "max-tries": 1}
     fixtureGpg = True
+    faketime = True
 
     def outputMatchPrepare(self, s):
         return re.sub(r'Signature made .* using', '', s)