diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index abff425f..9bfccf55 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,6 +18,8 @@ "UV_PROJECT_ENVIRONMENT": "/app/.venv" }, + "initializeCommand": "if docker info 2>/dev/null | grep -q rootless; then DS=\"${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/docker.sock\"; else DS=\"/var/run/docker.sock\"; fi; ln -sf \"$DS\" .docker.sock", + // The optional 'workspaceFolder' property is the path VS Code should open by default when // connected. This is typically a file mount in .devcontainer/docker-compose.yml "workspaceFolder": "/app", @@ -36,7 +38,7 @@ } }, - "postCreateCommand": "uv sync --dev --locked", + "postCreateCommand": "uv sync --dev", // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, diff --git a/.dockerignore b/.dockerignore index 319555f9..2b0a625d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,6 @@ **/dist **/node_modules .env -.flake8 .git .gitignore .github diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..42c152ba --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,205 @@ +name: CI +on: [push] +jobs: + pytest: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + # we use --locked instead of --frozen to elicit an error if pyproject.toml and uv.lock are not synced + run: uv sync --dev --locked + + - name: Run pytest + run: uv run pytest --ignore=test/availability + + - name: Generate badges + if: always() + run: | + PASSED=$(python -c "import xml.etree.ElementTree as ET; r=ET.parse('junit.xml').getroot(); ts=r.findall('testsuite'); print(sum(int(t.get('tests',0))-int(t.get('failures',0))-int(t.get('errors',0)) for t in ts))") + FAILED=$(python -c "import xml.etree.ElementTree as ET; r=ET.parse('junit.xml').getroot(); ts=r.findall('testsuite'); print(sum(int(t.get('failures',0))+int(t.get('errors',0)) for t in ts))") + if [ "$FAILED" -eq 0 ]; then + uv run anybadge --label=pytest --value="${PASSED} passed" --color=green --file=pytest.svg + else + uv run anybadge --label=pytest --value="${PASSED} passed, ${FAILED} failed" --color=red --file=pytest.svg + fi + COVERAGE=$(python -c "import xml.etree.ElementTree as ET; t=ET.parse('coverage.xml').getroot(); print(round(float(t.get('line-rate',0))*100))") + uv run anybadge --value=$COVERAGE --file=coverage.svg coverage + + - name: Upload pytest badge + if: always() + uses: actions/upload-artifact@v7 + with: + name: pytest + path: pytest.svg + overwrite: true + + - name: Upload coverage badge + if: always() + uses: actions/upload-artifact@v7 + with: + name: coverage + path: coverage.svg + overwrite: true + + + ty: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --dev --frozen + + - name: Run ty + run: uv run ty check + + - name: Generate badge + if: always() + run: | + if [ "${{ job.status }}" = "success" ]; then + uv run anybadge --label=ty --value=passing --color=green --file=ty.svg + else + uv run anybadge --label=ty --value=failing --color=red --file=ty.svg + fi + + - name: Upload ty badge + if: always() + uses: actions/upload-artifact@v7 + with: + name: ty + path: ty.svg + overwrite: true + + + ruff: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --dev --frozen + + - name: Lint with ruff + run: uv run ruff check --output-format=concise > ruff.txt + + - name: Generate badge + if: always() + run: | + VIOLATIONS=$(tail -1 ruff.txt | grep -oP '\d+' || echo 0) + if [ "$VIOLATIONS" -eq 0 ]; then + uv run anybadge --label=ruff --value=passing --color=green --file=ruff.svg + else + uv run anybadge --label=ruff --value="$VIOLATIONS violations" --color=orange --file=ruff.svg + fi + + - name: Upload ruff badge + if: always() + uses: actions/upload-artifact@v7 + with: + name: ruff + path: ruff.svg + overwrite: true + + + badges: + needs: [pytest, ty, ruff] + if: always() && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + with: + ref: badges + path: badges-branch + + - uses: actions/download-artifact@v7 + with: + name: pytest + path: badges-branch + + - uses: actions/download-artifact@v7 + with: + name: coverage + path: badges-branch + + - uses: actions/download-artifact@v7 + with: + name: ty + path: badges-branch + + - uses: actions/download-artifact@v7 + with: + name: ruff + path: badges-branch + + - name: Commit and push badges + working-directory: badges-branch + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add pytest.svg coverage.svg ty.svg ruff.svg + git diff --staged --quiet || git commit -m "ci: update badges" + git push + + + build: + needs: [pytest, ty, ruff] + if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Resolve build metadata + id: meta + run: | + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + echo "version=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT + echo "additional_tag=latest" >> $GITHUB_OUTPUT + elif [[ "$GITHUB_REF_NAME" == "beta" ]]; then + echo "version=beta" >> $GITHUB_OUTPUT + echo "additional_tag=" >> $GITHUB_OUTPUT + else + echo "version=dev" >> $GITHUB_OUTPUT + echo "additional_tag=" >> $GITHUB_OUTPUT + fi + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Build and push + run: | + docker buildx bake all --push \ + --set "*.cache-from=type=gha" \ + --set "*.cache-to=type=gha,mode=max" + env: + BUGHOG_VERSION: ${{ steps.meta.outputs.version }} + ADDITIONAL_TAG: ${{ steps.meta.outputs.additional_tag }} diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000..d75b2448 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,23 @@ +name: Create GitHub release + +on: + create: + +jobs: + package_release: + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + draft: true + prerelease: false diff --git a/.github/workflows/run-tests-and-linter.yml b/.github/workflows/run-tests-and-linter.yml deleted file mode 100644 index 4dd8c8e0..00000000 --- a/.github/workflows/run-tests-and-linter.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Pytest CI/CD -on: [push] -jobs: - pytest: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Install dependencies - run: uv sync --dev --locked - - - name: Run pytest - run: uv run pytest --ignore=test/availability - - - name: Generate badge - run: | - uv run genbadge tests -i junit.xml -o pytest.svg - uv run genbadge coverage -i coverage.xml -o coverage.svg - - - name: Upload test badge - uses: actions/upload-artifact@v4 - with: - name: pytest - path: pytest.svg - overwrite: true - - - name: Upload coverage badge - uses: actions/upload-artifact@v4 - with: - name: coverage - path: coverage.svg - overwrite: true - - - flake8: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - enable-cache: true - - - name: Install dependencies - run: | - uv sync --dev --locked - - - name: Lint with flake8 - run: | - uv run flake8 --exit-zero --output-file flake8.txt - - - name: Generate badge - run: | - uv run genbadge flake8 -i flake8.txt -o flake8.svg - - - name: Upload badge - uses: actions/upload-artifact@v4 - with: - name: flake8 - path: flake8.svg - overwrite: true diff --git a/.github/workflows/trigger-dev-build-job.yml b/.github/workflows/trigger-dev-build-job.yml deleted file mode 100644 index e40b76d0..00000000 --- a/.github/workflows/trigger-dev-build-job.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Trigger GitLab dev build job -on: - push: - branches: - - dev -env: - GITLAB_PIPELINE_TRIGGER_TOKEN: ${{ secrets.GITLAB_PIPELINE_TRIGGER_TOKEN }} -jobs: - trigger_build_job: - runs-on: ubuntu-latest - steps: - - name: send triggering request - run: | - curl -X POST \ - --fail \ - -F "token=$GITLAB_PIPELINE_TRIGGER_TOKEN" \ - -F "ref=main" \ - -F "variables[REPO]=${{github.repository}}" \ - https://gitlab.kuleuven.be/api/v4/projects/17581/trigger/pipeline diff --git a/.github/workflows/trigger-release-build-job.yml b/.github/workflows/trigger-release-build-job.yml deleted file mode 100644 index d1b682b7..00000000 --- a/.github/workflows/trigger-release-build-job.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Trigger GitLab release build job -on: - # Triggers the workflow on push or pull request events but only for the main branch - create: -env: - GITLAB_PIPELINE_TRIGGER_TOKEN: ${{ secrets.GITLAB_PIPELINE_TRIGGER_TOKEN }} -jobs: - - trigger_build_job: - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - runs-on: ubuntu-latest - steps: - - name: send triggering request - run: | - curl -X POST \ - --fail \ - -F "token=$GITLAB_PIPELINE_TRIGGER_TOKEN" \ - -F "ref=main" \ - -F "variables[REPO]=${{github.repository}}" \ - -F "variables[RELEASE_TAG]=${{github.ref_name}}" \ - https://gitlab.kuleuven.be/api/v4/projects/17581/trigger/pipeline - - package_release: - if: ${{ startsWith(github.ref, 'refs/tags/v') }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: ${{ github.ref }} - draft: true - prerelease: false diff --git a/.gitignore b/.gitignore index 2cca31d9..de2d382f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Docker socket symlink +.docker.sock + database/data/ nginx/ssl/certs/* diff --git a/.vscode/launch.json b/.vscode/launch.json index f8d0cfe3..ad29eccb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,10 +16,23 @@ ], "env": { "PYTHONPATH": "${workspaceFolder}", - "DISPLAY": ":1", - "MOZ_DISABLE_CONTENT_SANDBOX": "1", "PATH":"${PATH}:$HOME/.local/bin" } + }, + { + "name": "BugHog: Attach Worker (Dev Container)", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "bh_worker_0", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}/bughog", + "remoteRoot": "/app/bughog" + } + ] } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 0cb6b213..b9e39ed1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,15 +3,8 @@ "python.testing.pytestArgs": [ "test" ], - "python.testing.pytestEnabled": false, - "python.testing.unittestArgs": [ - "-v", - "-s", - "./test", - "-p", - "test_*.py" - ], - "python.testing.unittestEnabled": true, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.rulers": [100, 120], diff --git a/Dockerfile b/Dockerfile index 774dc87d..573fa742 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,12 +7,8 @@ RUN npm run build FROM openresty/openresty:1.27.1.1-3-bullseye AS nginx -RUN apt update -y && \ - apt install -y curl && \ - rm -rf /var/lib/apt/lists/* -RUN mkdir -p /www/data/js && \ - curl https://cdn.bokeh.org/bokeh/release/bokeh-3.6.1.min.js -o /www/data/js/bokeh.min.js && \ - curl https://cdn.bokeh.org/bokeh/release/bokeh-api-3.6.1.min.js -o /www/data/js/bokeh-api.min.js +ADD --chmod=755 https://cdn.bokeh.org/bokeh/release/bokeh-3.8.2.min.js /www/data/js/bokeh.min.js +ADD --chmod=755 https://cdn.bokeh.org/bokeh/release/bokeh-api-3.8.2.min.js /www/data/js/bokeh-api.min.js COPY ./nginx/start.sh /usr/local/bin/ COPY ./nginx/config /etc/nginx/config COPY --from=ui-build-stage /app/dist /www/data @@ -21,61 +17,51 @@ CMD ["start.sh"] FROM python:3.13-slim-bullseye AS base -COPY --from=ghcr.io/astral-sh/uv:0.9.7 /uv /uvx /bin/ + WORKDIR /app +ENV PATH="/app/.venv/bin:$PATH" -RUN apt-get update -RUN apt install -y curl gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libnspr4 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils libgbm-dev xvfb dbus-x11 libnss3-tools python3-tk python3-xlib gnome-screenshot vim git procps &&\ - rm -rf /var/lib/apt/lists/* -# Install Docker -RUN curl -sSL https://get.docker.com/ | sh - -# We manually install deprecated libraries for older browser versions. -# Most of them are Debian 11 (buster) packages. -# Stuff needed for chrome versions < 40 -RUN curl -sSLo multiarch-support.deb https://snapshot.debian.org/archive/debian-security/20240630T105336Z/pool/updates/main/g/glibc/multiarch-support_2.28-10%2Bdeb10u4_amd64.deb &&\ - curl -sSLo libgcrypt11.deb https://snapshot.debian.org/archive/debian/20130820T215153Z/pool/main/libg/libgcrypt11/libgcrypt11_1.5.3-2_amd64.deb &&\ - curl -sSLo libudev0.deb https://snapshot.debian.org/archive/debian/20111118T151858Z/pool/main/u/udev/libudev0_175-2_amd64.deb &&\ - curl -sSLo libpng12.deb https://snapshot.debian.org/archive/debian/20151118T214328Z/pool/main/libp/libpng/libpng12-0_1.2.54-1_amd64.deb &&\ - curl -sSLo pango1.0.deb https://snapshot.debian.org/archive/debian/20200504T084128Z/pool/main/p/pango1.0/libpango-1.0-0_1.42.4-8~deb10u1_amd64.deb &&\ - curl -sSLo libpangocairo-1.deb https://snapshot.debian.org/archive/debian/20210326T204420Z/pool/main/p/pango1.0/libpangocairo-1.0-0_1.42.4-8~deb10u1_amd64.deb &&\ - curl -sSLo libpangoft2.deb https://snapshot.debian.org/archive/debian/20210326T204420Z/pool/main/p/pango1.0/libpangoft2-1.0-0_1.42.4-8~deb10u1_amd64.deb &&\ - curl -sSLo libgtk-3.deb https://snapshot.debian.org/archive/debian/20210326T204420Z/pool/main/g/gtk+3.0/libgtk-3-0_3.24.5-1_amd64.deb &&\ - dpkg -i multiarch-support.deb &&\ - dpkg -i libgtk-3.deb libpangoft2.deb libpangocairo-1.deb pango1.0.deb libgcrypt11.deb libudev0.deb libpng12.deb &&\ - rm multiarch-support.deb libgtk-3.deb libpangoft2.deb libpangocairo-1.deb pango1.0.deb libgcrypt11.deb libudev0.deb libpng12.deb &&\ -# Stuff needed for chrome versions < 17 - ln -s /usr/lib/x86_64-linux-gnu/libnss3.so /usr/lib/x86_64-linux-gnu/libnss3.so.1d &&\ - ln -s /usr/lib/x86_64-linux-gnu/libnssutil3.so /usr/lib/x86_64-linux-gnu/libnssutil3.so.1d &&\ - ln -s /usr/lib/x86_64-linux-gnu/libsmime3.so /usr/lib/x86_64-linux-gnu/libsmime3.so.1d &&\ - ln -s /usr/lib/x86_64-linux-gnu/libssl3.so /usr/lib/x86_64-linux-gnu/libssl3.so.1d &&\ - ln -s /usr/lib/x86_64-linux-gnu/libplds4.so /usr/lib/x86_64-linux-gnu/libplds4.so.0d &&\ - ln -s /usr/lib/x86_64-linux-gnu/libplc4.so /usr/lib/x86_64-linux-gnu/libplc4.so.0d &&\ - ln -s /usr/lib/x86_64-linux-gnu/libnspr4.so /usr/lib/x86_64-linux-gnu/libnspr4.so.0d - -COPY subject/web_browser/profiles /app/subject/web_browser/profiles -COPY --chmod=0755 scripts/ /app/scripts/ -RUN cp /app/scripts/daemon/xvfb /etc/init.d/xvfb &&\ - mkdir -p /app/logs +FROM base AS python-app + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy + +COPY --from=ghcr.io/astral-sh/uv:0.11.7 /uv /bin/ -# Install python packages COPY pyproject.toml uv.lock /app/ -RUN uv sync --no-dev --locked -ENV PATH="/app/.venv/bin:$PATH" +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --no-dev --frozen --no-install-project -# Initiate PyAutoGUI -RUN touch /root/.Xauthority && \ - xauth add ${HOST}:0 . $(xxd -l 16 -p /dev/urandom) +COPY --chmod=0755 scripts/ /app/scripts/ +COPY bughog /app/bughog/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --no-dev --frozen FROM base AS core -# Copy rest of source code -COPY bughog /app/bughog + +# Install docker cli, and git for development container +RUN apt-get update && \ + apt-get install -y curl git gnupg && \ + curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/debian bullseye stable" \ + > /etc/apt/sources.list.d/docker.list && \ + apt-get update && \ + apt-get install -y docker-ce-cli && \ + apt-get remove -y --autoremove curl gnupg && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=python-app /bin/uv /bin/uv +COPY --from=python-app /app /app + ENTRYPOINT [ "/app/scripts/boot/core.sh" ] FROM base AS worker -# Copy rest of source code -COPY bughog /app/bughog + +COPY --from=python-app /bin/uv /bin/uv +COPY --from=python-app /app /app + ENTRYPOINT [ "/app/scripts/boot/worker.sh" ] diff --git a/README.md b/README.md index c2a59441..9cda3a06 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,20 @@
{{ banner_message }}
- - - - -{{ entry }}
-+ {{ banner_message }} +
+ + + + +{{ entry }}
+Project: {{ project_name }}
+PoC: {{ poc_name }}
+