From 8ffd2b6da107074a1e39a7dcec9e65253964f8a3 Mon Sep 17 00:00:00 2001 From: yazart Date: Mon, 15 Jun 2026 13:33:36 +0300 Subject: [PATCH] feat: build pipeline --- .DS_Store | Bin 0 -> 10244 bytes .gitignore | 3 +++ Dockerfile | 14 ++++++------ check.sh | 6 +++++ entrypoint.sh | 35 +++++++++++++++++++++++++++++ meta/angular.ng-template.yml | 23 +++++++++++++++---- meta/coverage_tools.yaml | 40 +++++++++++++++++++++++++++++++++ meta/eamodio.gitlens.yml | 2 +- pipe_listener.sh | 4 ++-- run.sh | 17 +++++++------- src/vsix_package_processing.py | 19 ++++++++++++++-- 11 files changed, 139 insertions(+), 24 deletions(-) create mode 100644 .DS_Store create mode 100755 entrypoint.sh diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..d36c14a0fb8e090826e56fa457372161de0c7054 GIT binary patch literal 10244 zcmeHMU2GIp6u#f|ZpfYui5z@`fku!YisEq}@OPx&vhE&ZXe>~=;vFr6tovn^O^ z8WSHx(fFkCFZ$#Ukp~qqzG$K-`lx7vF~(pt{yb=+FDeh7J9ie?784UutC>meJ@=e@ z&%O7Y`OciZcNt@7&FZTei!#P!Dx9hcRo8iVKQGfBDUk3^kUnExddwY3TTXh~*iaC9 zAoM`!fzSh?2SN|b3=dGw=8s$|Xjq3H2t5#b;ED(M_aRD!(|AB91@%82RQX2$qJ<>( zA8OM%z}JZeG#=1NL48%4Q=A?Us3JIGK&VrHln*Bw59p+zLY+aVGXyImIHAB_o%Evq zaE7>`VI6uP^uTlv@USUjCbQWH%jTWm&!$H#)Aj1>KY>(GSX5jhm&m2^zQm9-mhe(u z)@@B?_HwSvajbN%ol5U9jj^O!-sad|%CwD)uVCs14H@V$ZQU7bb+WeZ`nn0xAxGp$ zQmq^xU(;9{ZCJf-qBc6duA#m*x~B2AiHV3@5nHukck-|`Y&-W08xVX4*z74boya%H za!1?5g+xIjXV9lPgCes^mDyTPZ(qMMkW}Z5(#|_aJkzn=Jx0bQnpDeNvT)0wWADls zyuWAAv9iOqn^flxrmVrNmGX=h%SxDc8zd`p2Q!Xk?Q&d`Pt(`y8AmlqdPkEq*(aFbG@^$t42!(^fx z5waGoP210x-JWv2&Jn}rTRJcLZl}I}cPN!HI>!iu!H%|))PJQU=Sm&A={bHUd1Ggd zruFgZT6EpFVPVyRGA*HX%j$?C`Pono)DbL?IAG5ef-%YI_N04PQYW}y`2ScoO4#ac9>8SBuB zPVB;NBrt#>m^cI%M{x|}IF9>p0uSIM9>Sw|3Qyx1Jc}3cGG4(MyoNV%9`EA=e1uE* z5?|qK{D5Ea8!k&lQl(TSEs$!YTB%N2DK$yW(t2rwv{UModZmauR;_8?)N%!e0=GG+ zUK;>D>hog7N4<}VDTuw`$Dy)laW3$#hN1|~ZVDIUsLZ2QEeS@)mdF$?YEi&0TTbjY zwOFv#bt`2GO;z$)rMjUpLIJ221Z-Usu@H>1VPk|sQ7sJEMR8)8&s0~pchEHFRryBY z{35%=zF|MI%Y^een2##lf_g%CD>h@NkM9J!u^&ksKpHxRVZ*@@Li%w`;2uokUc&i< zg!G5;2p-21c#_cm9G=GuIE9x8>u2#g-oV><2j}o!fcnn}^WWk7DLCGdk7N9tkK@rq z#$AW1p$iz#Le%8w4wYj3#!l?>U`a3eBgo=IVj40B|dBmd4EIW^qEEJ;5T!z_87 zUxPC}OXeYYN}kM5sv$SiJy0Mv80Gi>9e@1)|4cU{++XN{&;x(L11M}yw71d(xnus+ zx%gdsl*$1r{Ndv!1@%>^@~`8G`q%Nq@O#B&RQTB3EeLPN1KFgYB%$XY|1;oEU$%wk M|M2|(|2+Ty4g$f3B>(^b literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore index e69de29..d562bfa 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,3 @@ +input/* +report/* +status/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2d465a2..565fc5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN apt-get update -y && \ setcap cap_net_raw,cap_net_admin=eip /bin/tcpdump # Install Chrome browser for Karma -RUN cd /tmp && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt-get update -y && apt-get install -y ./google-chrome-stable_current_amd64.deb && rm -f google-chrome-stable_current_amd64.deb +# RUN cd /tmp && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt-get update -y && apt-get install -y ./google-chrome-stable_current_amd64.deb && rm -f google-chrome-stable_current_amd64.deb RUN if [ ${LOCAL_BUILD} -eq 1 ]; then groupadd nandy; else groupadd -g "${DST_GROUP_ID}" nandy; fi RUN useradd -d /home/nandy -m -s /bin/bash -u "${DST_USER_ID}" -g "${DST_GROUP_ID}" nandy; @@ -38,13 +38,14 @@ ENV NVM_DIR="$HOME/.nvm" RUN echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> $HOME/.profile ENV NVM_DIR $HOME/.nvm -ENV NODE_VERSION 20.16.0 +ENV NODE_VERSION 22.18.0 -RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.40.0/install.sh | bash \ +RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.40.0/install.sh | env -u NODE_VERSION bash \ && . $NVM_DIR/nvm.sh \ && nvm install $NODE_VERSION \ && nvm alias default $NODE_VERSION \ - && nvm use default + && nvm use default \ + && npm install --global yarn@1.22.22 # Fix git issue with big resps RUN git config --global http.postBuffer 52428800 @@ -68,9 +69,8 @@ RUN source $PROCESSING_VENV/bin/activate && pip install $WORKDIR/runtime_check/ COPY --chown=nandy:nandy ./scanfeeder /usr/local/bin/scanfeeder -COPY --chown=nandy:nandy ./src/npm_package_processing.py $WORKDIR -COPY --chown=nandy:nandy ./src/debug_check.py $WORKDIR -COPY --chown=nandy:nandy ./metadata $HOME/metadata +COPY --chown=nandy:nandy ./src/vsix_package_processing.py $WORKDIR +COPY --chown=nandy:nandy ./meta $HOME/meta COPY --chown=nandy:nandy ./run.sh /usr/local/bin/run.sh RUN chmod +x /usr/local/bin/run.sh COPY --chown=nandy ./pipe_create.sh /usr/local/bin/pipe_create.sh diff --git a/check.sh b/check.sh index 8d2bba1..4703e1e 100755 --- a/check.sh +++ b/check.sh @@ -38,6 +38,12 @@ curl \ --output "input/${RESULT_INPUT}" \ "${URL}" +if gzip -t "input/${RESULT_INPUT}" 2>/dev/null; then + echo "Decompress gzip-wrapped VSIX" + gzip -dc "input/${RESULT_INPUT}" > "input/${RESULT_INPUT}.tmp" + mv "input/${RESULT_INPUT}.tmp" "input/${RESULT_INPUT}" +fi + export RELOAD_CONFIG=1 export ADD_METADATA_VOLUME='true' bash ./run.sh "${RESULT_INPUT}" "$PACKAGE_NAME" "$PACKAGE_VERSION" diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..6f81e77 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e + +update_user_ids() { + local target_uid="${DST_USER_ID}" + local target_gid="${DST_GROUP_ID}" + + if [ -z "$target_uid" ] || [ -z "$target_gid" ]; then + echo "Warning: DST_USER_ID or DST_GROUP_ID not set, using default values" + return 0 + fi + + local current_uid=$(id -u nandy) + local current_gid=$(id -g nandy) + + if [ "$current_uid" -eq "$target_uid" ] && [ "$current_gid" -eq "$target_gid" ]; then + echo "User nandy already has UID=$target_uid and GID=$target_gid" + return 0 + fi + + echo "Updating nandy user: UID $current_uid -> $target_uid, GID $current_gid -> $target_gid" + groupmod -g "$target_gid" nandy + usermod -u "$target_uid" nandy + find /home/nandy -user "$current_uid" -exec chown -h "$target_uid" {} \; 2>/dev/null || true + find /home/nandy -group "$current_gid" -exec chgrp -h "$target_gid" {} \; 2>/dev/null || true + echo "User IDs updated successfully" +} +update_user_ids + +PLUGIN_VERSION=${PLUGIN_VERSION:-"unknown"} +PLUGIN_BUILD_DATE=${PLUGIN_BUILD_DATE:-"unknown"} +echo "Plugin version: ${PLUGIN_VERSION}" +echo "Plugin build date: ${PLUGIN_BUILD_DATE}" + +exec runuser -u nandy -- "$@" diff --git a/meta/angular.ng-template.yml b/meta/angular.ng-template.yml index 886ecaa..7c26804 100644 --- a/meta/angular.ng-template.yml +++ b/meta/angular.ng-template.yml @@ -1,11 +1,26 @@ # The VSIX manifest points at angular/angular, but the VS Code extension source # lives in angular/vscode-ng-language-service. There is no v20.3.3 tag there; # v20.3.0 is the nearest public tag for the 20.3 line. -install_dependencies: "corepack enable && corepack prepare yarn@1.22.22 --activate && yarn install --frozen-lockfile" +install_dependencies: "yarn install --frozen-lockfile" source_repository: "https://github.com/angular/vscode-ng-language-service.git" source_ref: "v20.3.0" -build_vsix: "yarn package" -run_tests: "yarn test" +build_vsix: + - "yarn package" + - "mv bazel-bin/ng-template.vsix ./ng-template.vsix" + - unzip -o ng-template.vsix -d ng-template + - tail -n +5 ng-template/extension/CHANGELOG.md > temp.md && mv temp.md ng-template/extension/CHANGELOG.md + - sed 's|bazel-out/k8-fastbuild|bazel-out/darwin_arm64-fastbuild|g' ng-template/extension/index.js > ng-template/extension/indexx.js + - mv ng-template/extension/indexx.js ng-template/extension/index.js + - sed 's|../report/git_package/angular.ng-template|Users/atscott/github/vscode-ng-language-service|g' ng-template/extension/index.js > ng-template/extension/indexx.js + - mv ng-template/extension/indexx.js ng-template/extension/index.js + - sed 's|bazel-out/k8-fastbuild|bazel-out/darwin_arm64-fastbuild|g' ng-template/extension/server/index.js > ng-template/extension/server/indexx.js + - mv ng-template/extension/server/indexx.js ng-template/extension/server/index.js + - sed 's|../report/git_package/angular.ng-template|Users/atscott/github/vscode-ng-language-service|g' ng-template/extension/server/index.js > ng-template/extension/server/indexx.js + - mv ng-template/extension/server/indexx.js ng-template/extension/server/index.js + - cd ng-template && python3 -m zipfile -c ../ng-templateX.vsix . && cd .. +built_vsix_path: "ng-templateX.vsix" +run_tests: "npx nyc yarn compile:test && node_modules/.bin/jasmine --config=jasmine.json" coverage: - enabled: false + tool: "nyc" + enabled: true threshold: 80 diff --git a/meta/coverage_tools.yaml b/meta/coverage_tools.yaml index e69de29..a975e0c 100644 --- a/meta/coverage_tools.yaml +++ b/meta/coverage_tools.yaml @@ -0,0 +1,40 @@ +c8: + report: + - npx c8 report --reporter=text + check: + - npx c8 check-coverage --branches 80 --functions 80 --lines 80 --statements 80 + coverage_report_folder: coverage +vitest: + report: + - npx vitest --coverage --run + check: + - npx vitest --run --coverage --coverage.thresholds.lines=80 --coverage.thresholds.functions=80 --coverage.thresholds.branches=80 --coverage.thresholds.statements=80 + coverage_report_folder: coverage +nyc: + report: + - npx nyc report + check: + - npx nyc check-coverage --branches 80 --functions 80 --lines 80 + --statements 80 + coverage_report_folder: .nyc_output +jest: + report: + - npx istanbul report text + check: + - npx istanbul check-coverage --statements 80 --functions 80 --branches 80 --lines 80 + coverage_report_folder: coverage +istanbul: + report: + - npx istanbul report text + check: + - npx istanbul check-coverage --statements 80 --functions 80 --branches 80 --lines 80 + coverage_report_folder: coverage +tap: + report: + - npx tap report --coverage-report=text --show-full-coverage + check: + - "" + coverage_report_folder: .tap/coverage +karma: + report: + - exit 2 diff --git a/meta/eamodio.gitlens.yml b/meta/eamodio.gitlens.yml index 9b18a6c..77b3ebd 100644 --- a/meta/eamodio.gitlens.yml +++ b/meta/eamodio.gitlens.yml @@ -1,5 +1,5 @@ # Commands are executed from the cloned repository root. -install_dependencies: "node -e 'const [major, minor] = process.versions.node.split(`.`).map(Number); if (major < 22 || (major === 22 && minor < 18)) { console.error(`GitLens 17.12.1 requires Node >=22.18.0; current=${process.versions.node}`); process.exit(1); }' && corepack enable && corepack prepare pnpm@10.33.0 --activate && pnpm install --frozen-lockfile" +install_dependencies: "npm i -g c8 && node -e 'const [major, minor] = process.versions.node.split(`.`).map(Number); if (major < 22 || (major === 22 && minor < 18)) { console.error(`GitLens 17.12.1 requires Node >=22.18.0; current=${process.versions.node}`); process.exit(1); }' && corepack enable && corepack prepare pnpm@10.33.0 --activate && pnpm install --frozen-lockfile" build_vsix: "pnpm exec vsce package --no-dependencies --out source.vsix" run_tests: "npx c8 --clean --temp-directory coverage/tmp node_modules/.bin/vscode-test" coverage: diff --git a/pipe_listener.sh b/pipe_listener.sh index 6b314fb..2d41bfb 100755 --- a/pipe_listener.sh +++ b/pipe_listener.sh @@ -37,7 +37,7 @@ do continue fi - CONTAINER_ID=$(docker ps -a | grep npm_runtime_check | awk '{print $1;}') + CONTAINER_ID=$(docker ps -a | grep vsix_runtime_check | awk '{print $1;}') SYSDIG_FILE=$(awk '{print $3}' <<< ${STR}) if [ "${CONTAINER_ID}" == "" ]; then @@ -91,4 +91,4 @@ do exit 0 ;; esac -done \ No newline at end of file +done diff --git a/run.sh b/run.sh index 6297595..9133e60 100755 --- a/run.sh +++ b/run.sh @@ -38,7 +38,7 @@ MINIO_USER=${10:-"admin"} MINIO_PASSWORD=${11:-"password"} # MinIO bucket name, default is MINIO_BUCKET=${12:-"taskalo-files"} -PLUGIN_NAME="npm_runtime_check" +PLUGIN_NAME="vsix_runtime_check" SCANFEEDER_DEBUG=${SCANFEEDER_DEBUG:-true} USE_SYSDIG=${USE_SYSDIG:-false} DST_USER_ID=$(id -u) @@ -69,15 +69,16 @@ fi MONITOR_ENABLE=$([ "${USE_SYSDIG}" = "true" ] && echo "True" || echo "False") USER_HOME='/home/nandy' -ADD_METADATA_VOLUME=$([ "${ADD_METADATA_VOLUME}" = "true" ] && echo "-v ${PWD}/metadata:${USER_HOME}/metadata" || \ +ADD_METADATA_VOLUME=$([ "${ADD_METADATA_VOLUME}" = "true" ] && echo "-v ${PWD}/meta:${USER_HOME}/meta" || \ echo "") -NPM_RUNTIME_CHECK_IMAGE=${NPM_RUNTIME_CHECK_IMAGE:-npm_runtime_check} +VSIX_RUNTIME_CHECK_IMAGE=${VSIX_RUNTIME_CHECK_IMAGE:-vsix_runtime_check} -echo "=====================> start ${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest}" +echo "=====================> start ${VSIX_RUNTIME_CHECK_IMAGE}:${VSIX_RUNTIME_CHECK_VER:-latest}" docker run --rm -i \ + --platform linux/amd64 \ --network ${NETWORK} \ --add-host=host.docker.internal:host-gateway \ --privileged \ @@ -92,7 +93,7 @@ docker run --rm -i \ -e PACKAGE_FILE=$PACKAGE_FILE \ -e PACKAGE_NAME=$PACKAGE_NAME \ -e PACKAGE_VERSION=$PACKAGE_VERSION \ - -e METADATA_PATH="${USER_HOME}/metadata" \ + -e METADATA_PATH="${USER_HOME}/meta" \ -e PLUGIN_NAME="${PLUGIN_NAME}" \ -e PIPELINE_ID=${PIPELINE_ID} \ -e SCANHUB_URL=${SCANHUB_URL} \ @@ -109,11 +110,11 @@ docker run --rm -i \ -e LOG_LEVEL="DEBUG" \ -e T05_DEFAULT_TIMEOUT=1800 \ -e SCANFEEDER_PATH="/usr/local/bin/scanfeeder" \ - "${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest}" bash -l -c "source ${USER_HOME}/processing/venv/bin/activate && python3 ${USER_HOME}/processing/npm_package_processing.py --file_path=/input/${PACKAGE_FILE} --report_path=/report" + "${VSIX_RUNTIME_CHECK_IMAGE}:${VSIX_RUNTIME_CHECK_VER:-latest}" bash -l -c "source ${USER_HOME}/processing/venv/bin/activate && python3 ${USER_HOME}/processing/vsix_package_processing.py --file_path=/input/${PACKAGE_FILE} --report_path=/report" RES_PLUGIN=$? -echo "=====================> finished ${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest} with code ${RES_PLUGIN}" +echo "=====================> finished ${VSIX_RUNTIME_CHECK_IMAGE}:${VSIX_RUNTIME_CHECK_VER:-latest} with code ${RES_PLUGIN}" finish_with_pipe -echo "Npm runtime check is done, exit with code ${RES_PLUGIN}" +echo "VSIX runtime check is done, exit with code ${RES_PLUGIN}" exit ${RES_PLUGIN} diff --git a/src/vsix_package_processing.py b/src/vsix_package_processing.py index c5cc7a3..e5abed7 100644 --- a/src/vsix_package_processing.py +++ b/src/vsix_package_processing.py @@ -1,11 +1,13 @@ #!/usr/bin/python3 import os +import gzip import json import logging import os.path import pathlib import argparse +import shutil import tempfile import subprocess import zipfile @@ -42,7 +44,7 @@ class VSIX_T5Checker(T5Checker): yaml_specific_version = dict() stage_name='load_yaml_config' - metadata_root = pathlib.Path(settings.get("metadata_path", "../metadata")) + metadata_root = pathlib.Path(settings.get("metadata_path", "../meta")) coverage_tool_paths = ( metadata_root / "coverage_tools.yaml", pathlib.Path("../meta/coverage_tools.yaml"), @@ -141,12 +143,25 @@ class VSIX_T5Checker(T5Checker): try: result_folder.mkdir(parents=True, exist_ok=False) result_root = result_folder.resolve() - with zipfile.ZipFile(archive_path) as archive: + + archive_source = archive_path + decompressed_path = None + with archive_path.open("rb") as package_file: + is_gzip_wrapped = package_file.read(2) == b"\x1f\x8b" + if is_gzip_wrapped: + decompressed_path = result_folder.parent / f"{archive_path.name}.decompressed" + with gzip.open(archive_path, "rb") as source, decompressed_path.open("wb") as output: + shutil.copyfileobj(source, output) + archive_source = decompressed_path + + with zipfile.ZipFile(archive_source) as archive: for member in archive.infolist(): destination = (result_folder / member.filename).resolve() if destination != result_root and result_root not in destination.parents: raise ValueError(f"Unsafe path in VSIX archive: {member.filename}") archive.extractall(result_folder) + if decompressed_path: + decompressed_path.unlink() except (OSError, ValueError, zipfile.BadZipFile) as e: self.stage_error( stage_name=substep_name,