Compare commits

...
4 Commits
Author SHA1 Message Date
yazart 7c62a4f62c chore: add git ignore DS_Store 2026-06-15 13:35:15 +03:00
yazart aa237cce0c chore: delete DS_Store 2026-06-15 13:34:18 +03:00
yazart 8ffd2b6da1 feat: build pipeline 2026-06-15 13:33:36 +03:00
yazart de65873e45 chore: fix to vsix 2026-06-10 17:23:29 +03:00
20 changed files with 894 additions and 42 deletions
+4
View File
@@ -0,0 +1,4 @@
input/*
report/*
status/*
.DS_Store
+7 -7
View File
@@ -21,7 +21,7 @@ RUN apt-get update -y && \
setcap cap_net_raw,cap_net_admin=eip /bin/tcpdump
# Install Chrome browser for Karma
RUN cd /tmp && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt-get update -y && apt-get install -y ./google-chrome-stable_current_amd64.deb && rm -f google-chrome-stable_current_amd64.deb
# RUN cd /tmp && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && apt-get update -y && apt-get install -y ./google-chrome-stable_current_amd64.deb && rm -f google-chrome-stable_current_amd64.deb
RUN if [ ${LOCAL_BUILD} -eq 1 ]; then groupadd nandy; else groupadd -g "${DST_GROUP_ID}" nandy; fi
RUN useradd -d /home/nandy -m -s /bin/bash -u "${DST_USER_ID}" -g "${DST_GROUP_ID}" nandy;
@@ -38,13 +38,14 @@ ENV NVM_DIR="$HOME/.nvm"
RUN echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >> $HOME/.profile
ENV NVM_DIR $HOME/.nvm
ENV NODE_VERSION 20.16.0
ENV NODE_VERSION 22.18.0
RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.40.0/install.sh | bash \
RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.40.0/install.sh | env -u NODE_VERSION bash \
&& . $NVM_DIR/nvm.sh \
&& nvm install $NODE_VERSION \
&& nvm alias default $NODE_VERSION \
&& nvm use default
&& nvm use default \
&& npm install --global yarn@1.22.22
# Fix git issue with big resps
RUN git config --global http.postBuffer 52428800
@@ -68,9 +69,8 @@ RUN source $PROCESSING_VENV/bin/activate && pip install $WORKDIR/runtime_check/
COPY --chown=nandy:nandy ./scanfeeder /usr/local/bin/scanfeeder
COPY --chown=nandy:nandy ./src/npm_package_processing.py $WORKDIR
COPY --chown=nandy:nandy ./src/debug_check.py $WORKDIR
COPY --chown=nandy:nandy ./metadata $HOME/metadata
COPY --chown=nandy:nandy ./src/vsix_package_processing.py $WORKDIR
COPY --chown=nandy:nandy ./meta $HOME/meta
COPY --chown=nandy:nandy ./run.sh /usr/local/bin/run.sh
RUN chmod +x /usr/local/bin/run.sh
COPY --chown=nandy ./pipe_create.sh /usr/local/bin/pipe_create.sh
+33 -25
View File
@@ -1,40 +1,48 @@
#!/bin/bash
mkdir input report output
export NPM_RUNTIME_CHECK_VER=latest
PACKAGE_NAME=${1}
PACKAGE_VERSION=${2}
set -e
RESULT_NAME=$PACKAGE_NAME
if [ ${PACKAGE_NAME:0:1} == '@' ]
then
POS=$(echo $PACKAGE_NAME | grep -ob "/" | grep -oE "[0-9]+" | head -n 1)
RESULT_NAME=${PACKAGE_NAME:$POS+1}
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <publisher.extension> <version>"
exit 1
fi
URL=$(npm view ${PACKAGE_NAME}@${PACKAGE_VERSION} dist.tarball)
PACKAGE_NAME=$1
PACKAGE_VERSION=$2
REPORT=$(ls ./report/git_package)
REPORT_LEN=${#REPORT}
if [ ${REPORT_LEN} != 0 ]
then
if [[ "$PACKAGE_NAME" != *.* ]]; then
echo "Package name must use publisher.extension format"
exit 1
fi
PUBLISHER=${PACKAGE_NAME%%.*}
EXTENSION=${PACKAGE_NAME#*.}
URL="https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${PUBLISHER}/vsextensions/${EXTENSION}/${PACKAGE_VERSION}/vspackage"
RESULT_INPUT="${PACKAGE_NAME}-${PACKAGE_VERSION}.vsix"
mkdir -p input report output
export VSIX_RUNTIME_CHECK_VER=${VSIX_RUNTIME_CHECK_VER:-latest}
if [ -d ./report/git_package ]; then
sync
rm -rf ./report/git_package
echo "Clean git_package"
fi
RESULT_INPUT=${RESULT_NAME}-${PACKAGE_VERSION}.tgz
echo "Download ${PACKAGE_NAME}@${PACKAGE_VERSION} from ${URL}"
curl \
--fail \
--location \
--header "Accept: application/octet-stream" \
--header "Accept-Encoding: identity" \
--output "input/${RESULT_INPUT}" \
"${URL}"
cd input
if [ $(ls $RESULT_INPUT) ]
then
rm $RESULT_INPUT
fi
set -x
echo $URL
wget -O "${RESULT_INPUT}" "${URL}"
cd ..
if gzip -t "input/${RESULT_INPUT}" 2>/dev/null; then
echo "Decompress gzip-wrapped VSIX"
gzip -dc "input/${RESULT_INPUT}" > "input/${RESULT_INPUT}.tmp"
mv "input/${RESULT_INPUT}.tmp" "input/${RESULT_INPUT}"
fi
export RELOAD_CONFIG=1
export ADD_METADATA_VOLUME='true'
Executable
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
set -e
update_user_ids() {
local target_uid="${DST_USER_ID}"
local target_gid="${DST_GROUP_ID}"
if [ -z "$target_uid" ] || [ -z "$target_gid" ]; then
echo "Warning: DST_USER_ID or DST_GROUP_ID not set, using default values"
return 0
fi
local current_uid=$(id -u nandy)
local current_gid=$(id -g nandy)
if [ "$current_uid" -eq "$target_uid" ] && [ "$current_gid" -eq "$target_gid" ]; then
echo "User nandy already has UID=$target_uid and GID=$target_gid"
return 0
fi
echo "Updating nandy user: UID $current_uid -> $target_uid, GID $current_gid -> $target_gid"
groupmod -g "$target_gid" nandy
usermod -u "$target_uid" nandy
find /home/nandy -user "$current_uid" -exec chown -h "$target_uid" {} \; 2>/dev/null || true
find /home/nandy -group "$current_gid" -exec chgrp -h "$target_gid" {} \; 2>/dev/null || true
echo "User IDs updated successfully"
}
update_user_ids
PLUGIN_VERSION=${PLUGIN_VERSION:-"unknown"}
PLUGIN_BUILD_DATE=${PLUGIN_BUILD_DATE:-"unknown"}
echo "Plugin version: ${PLUGIN_VERSION}"
echo "Plugin build date: ${PLUGIN_BUILD_DATE}"
exec runuser -u nandy -- "$@"
+26
View File
@@ -0,0 +1,26 @@
# The VSIX manifest points at angular/angular, but the VS Code extension source
# lives in angular/vscode-ng-language-service. There is no v20.3.3 tag there;
# v20.3.0 is the nearest public tag for the 20.3 line.
install_dependencies: "yarn install --frozen-lockfile"
source_repository: "https://github.com/angular/vscode-ng-language-service.git"
source_ref: "v20.3.0"
build_vsix:
- "yarn package"
- "mv bazel-bin/ng-template.vsix ./ng-template.vsix"
- unzip -o ng-template.vsix -d ng-template
- tail -n +5 ng-template/extension/CHANGELOG.md > temp.md && mv temp.md ng-template/extension/CHANGELOG.md
- sed 's|bazel-out/k8-fastbuild|bazel-out/darwin_arm64-fastbuild|g' ng-template/extension/index.js > ng-template/extension/indexx.js
- mv ng-template/extension/indexx.js ng-template/extension/index.js
- sed 's|../report/git_package/angular.ng-template|Users/atscott/github/vscode-ng-language-service|g' ng-template/extension/index.js > ng-template/extension/indexx.js
- mv ng-template/extension/indexx.js ng-template/extension/index.js
- sed 's|bazel-out/k8-fastbuild|bazel-out/darwin_arm64-fastbuild|g' ng-template/extension/server/index.js > ng-template/extension/server/indexx.js
- mv ng-template/extension/server/indexx.js ng-template/extension/server/index.js
- sed 's|../report/git_package/angular.ng-template|Users/atscott/github/vscode-ng-language-service|g' ng-template/extension/server/index.js > ng-template/extension/server/indexx.js
- mv ng-template/extension/server/indexx.js ng-template/extension/server/index.js
- cd ng-template && python3 -m zipfile -c ../ng-templateX.vsix . && cd ..
built_vsix_path: "ng-templateX.vsix"
run_tests: "npx nyc yarn compile:test && node_modules/.bin/jasmine --config=jasmine.json"
coverage:
tool: "nyc"
enabled: true
threshold: 80
+40
View File
@@ -0,0 +1,40 @@
c8:
report:
- npx c8 report --reporter=text
check:
- npx c8 check-coverage --branches 80 --functions 80 --lines 80 --statements 80
coverage_report_folder: coverage
vitest:
report:
- npx vitest --coverage --run
check:
- npx vitest --run --coverage --coverage.thresholds.lines=80 --coverage.thresholds.functions=80 --coverage.thresholds.branches=80 --coverage.thresholds.statements=80
coverage_report_folder: coverage
nyc:
report:
- npx nyc report
check:
- npx nyc check-coverage --branches 80 --functions 80 --lines 80
--statements 80
coverage_report_folder: .nyc_output
jest:
report:
- npx istanbul report text
check:
- npx istanbul check-coverage --statements 80 --functions 80 --branches 80 --lines 80
coverage_report_folder: coverage
istanbul:
report:
- npx istanbul report text
check:
- npx istanbul check-coverage --statements 80 --functions 80 --branches 80 --lines 80
coverage_report_folder: coverage
tap:
report:
- npx tap report --coverage-report=text --show-full-coverage
check:
- ""
coverage_report_folder: .tap/coverage
karma:
report:
- exit 2
+9
View File
@@ -0,0 +1,9 @@
# Marketplace version 3.0.19 is tagged as pre-release/3.0.19 upstream.
install_dependencies: "npm ci"
source_ref: "pre-release/3.0.19"
build_vsix: "npx @vscode/vsce package --out source.vsix"
run_tests: "npx c8 --clean --temp-directory coverage/tmp npm test"
coverage:
enabled: true
tool: "c8"
threshold: 80
+8
View File
@@ -0,0 +1,8 @@
# Commands are executed from the cloned repository root.
install_dependencies: "npm i -g c8 && node -e 'const [major, minor] = process.versions.node.split(`.`).map(Number); if (major < 22 || (major === 22 && minor < 18)) { console.error(`GitLens 17.12.1 requires Node >=22.18.0; current=${process.versions.node}`); process.exit(1); }' && corepack enable && corepack prepare pnpm@10.33.0 --activate && pnpm install --frozen-lockfile"
build_vsix: "pnpm exec vsce package --no-dependencies --out source.vsix"
run_tests: "npx c8 --clean --temp-directory coverage/tmp node_modules/.bin/vscode-test"
coverage:
enabled: true
tool: "c8"
threshold: 80
+6
View File
@@ -0,0 +1,6 @@
install_dependencies: "npm ci"
build_vsix: "npx vsce package --out source.vsix"
run_tests: "npm test"
coverage:
enabled: false
threshold: 80
+6
View File
@@ -0,0 +1,6 @@
install_dependencies: "corepack enable && corepack prepare yarn@1.22.22 --activate && yarn install --frozen-lockfile"
build_vsix: "npx vsce package --out source.vsix"
run_tests: "yarn test"
coverage:
enabled: false
threshold: 80
+7
View File
@@ -0,0 +1,7 @@
install_dependencies: "npm ci"
build_vsix: "npx vsce package --out source.vsix"
run_tests: "npm run build-bundle && npm run jest -- --coverage"
coverage:
enabled: true
tool: "jest"
threshold: 80
+2 -2
View File
@@ -37,7 +37,7 @@ do
continue
fi
CONTAINER_ID=$(docker ps -a | grep npm_runtime_check | awk '{print $1;}')
CONTAINER_ID=$(docker ps -a | grep vsix_runtime_check | awk '{print $1;}')
SYSDIG_FILE=$(awk '{print $3}' <<< ${STR})
if [ "${CONTAINER_ID}" == "" ]; then
@@ -91,4 +91,4 @@ do
exit 0
;;
esac
done
done
+9 -8
View File
@@ -38,7 +38,7 @@ MINIO_USER=${10:-"admin"}
MINIO_PASSWORD=${11:-"password"}
# MinIO bucket name, default is
MINIO_BUCKET=${12:-"taskalo-files"}
PLUGIN_NAME="npm_runtime_check"
PLUGIN_NAME="vsix_runtime_check"
SCANFEEDER_DEBUG=${SCANFEEDER_DEBUG:-true}
USE_SYSDIG=${USE_SYSDIG:-false}
DST_USER_ID=$(id -u)
@@ -69,15 +69,16 @@ fi
MONITOR_ENABLE=$([ "${USE_SYSDIG}" = "true" ] && echo "True" || echo "False")
USER_HOME='/home/nandy'
ADD_METADATA_VOLUME=$([ "${ADD_METADATA_VOLUME}" = "true" ] && echo "-v ${PWD}/metadata:${USER_HOME}/metadata" || \
ADD_METADATA_VOLUME=$([ "${ADD_METADATA_VOLUME}" = "true" ] && echo "-v ${PWD}/meta:${USER_HOME}/meta" || \
echo "")
NPM_RUNTIME_CHECK_IMAGE=${NPM_RUNTIME_CHECK_IMAGE:-npm_runtime_check}
VSIX_RUNTIME_CHECK_IMAGE=${VSIX_RUNTIME_CHECK_IMAGE:-vsix_runtime_check}
echo "=====================> start ${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest}"
echo "=====================> start ${VSIX_RUNTIME_CHECK_IMAGE}:${VSIX_RUNTIME_CHECK_VER:-latest}"
docker run --rm -i \
--platform linux/amd64 \
--network ${NETWORK} \
--add-host=host.docker.internal:host-gateway \
--privileged \
@@ -92,7 +93,7 @@ docker run --rm -i \
-e PACKAGE_FILE=$PACKAGE_FILE \
-e PACKAGE_NAME=$PACKAGE_NAME \
-e PACKAGE_VERSION=$PACKAGE_VERSION \
-e METADATA_PATH="${USER_HOME}/metadata" \
-e METADATA_PATH="${USER_HOME}/meta" \
-e PLUGIN_NAME="${PLUGIN_NAME}" \
-e PIPELINE_ID=${PIPELINE_ID} \
-e SCANHUB_URL=${SCANHUB_URL} \
@@ -109,11 +110,11 @@ docker run --rm -i \
-e LOG_LEVEL="DEBUG" \
-e T05_DEFAULT_TIMEOUT=1800 \
-e SCANFEEDER_PATH="/usr/local/bin/scanfeeder" \
"${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest}" bash -l -c "source ${USER_HOME}/processing/venv/bin/activate && python3 ${USER_HOME}/processing/npm_package_processing.py --file_path=/input/${PACKAGE_FILE} --report_path=/report"
"${VSIX_RUNTIME_CHECK_IMAGE}:${VSIX_RUNTIME_CHECK_VER:-latest}" bash -l -c "source ${USER_HOME}/processing/venv/bin/activate && python3 ${USER_HOME}/processing/vsix_package_processing.py --file_path=/input/${PACKAGE_FILE} --report_path=/report"
RES_PLUGIN=$?
echo "=====================> finished ${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest} with code ${RES_PLUGIN}"
echo "=====================> finished ${VSIX_RUNTIME_CHECK_IMAGE}:${VSIX_RUNTIME_CHECK_VER:-latest} with code ${RES_PLUGIN}"
finish_with_pipe
echo "Npm runtime check is done, exit with code ${RES_PLUGIN}"
echo "VSIX runtime check is done, exit with code ${RES_PLUGIN}"
exit ${RES_PLUGIN}
Executable
+1
View File
@@ -0,0 +1 @@
#!/bin/bash
+4
View File
@@ -0,0 +1,4 @@
GitPython
PyYAML
wget
scapy
+652
View File
@@ -0,0 +1,652 @@
#!/usr/bin/python3
import os
import gzip
import json
import logging
import os.path
import pathlib
import argparse
import shutil
import tempfile
import subprocess
import zipfile
from urllib.parse import urlparse
import yaml
from runtime_check.checker import T5Checker, Scanfeeder
from runtime_check.logger import get_logger
from runtime_check.monitor import Monitor
from runtime_check.config import settings
MANUAL_PROCESSING_KEY = 'MANUAL_CHECK'
MANUAL_CHECK_DETECTION_MESSAGE = 'MANUAL_CHECK_DETECTION'
class VSIX_T5Checker(T5Checker):
"""Base class to perform the whole check. We may inherit it and overwrite any part in some specific case, so don't hesitate"""
def __init__(
self,
scanfeeder: Scanfeeder,
monitor: Monitor,
logger: logging.Logger,
report_dir: pathlib.Path,
package_path: pathlib.Path,
):
self.package_path = pathlib.Path(package_path)
super().__init__(scanfeeder, monitor, logger, report_dir)
def _load_yaml_config(self) -> None:
self.packages_info = dict()
yaml_base_info = dict()
yaml_specific_version = dict()
stage_name='load_yaml_config'
metadata_root = pathlib.Path(settings.get("metadata_path", "../meta"))
coverage_tool_paths = (
metadata_root / "coverage_tools.yaml",
pathlib.Path("../meta/coverage_tools.yaml"),
)
coverage_tool_path = next(
(path for path in coverage_tool_paths if path.exists()),
None,
)
if coverage_tool_path:
with coverage_tool_path.open(encoding="utf-8") as coverage_file:
self.supported_coverage_tools = yaml.load(
coverage_file,
Loader=yaml.SafeLoader,
) or {}
else:
self.stage_error(
stage_name=stage_name,
details=(
"Failed to load supported coverage tools yaml file from "
+ ", ".join(str(path) for path in coverage_tool_paths)
),
)
package_metadata_paths = (
metadata_root / settings.package_name / "metadata.yaml",
metadata_root / f"{settings.package_name}.yml",
metadata_root / f"{settings.package_name}.yaml",
)
package_metadata_path = next(
(path for path in package_metadata_paths if path.exists()),
None,
)
package_specific_version = (
metadata_root
/ settings.package_name
/ settings.package_version
/ "metadata.yaml"
)
if package_metadata_path:
with package_metadata_path.open(encoding="utf-8") as metadata_file:
yaml_base_info = yaml.load(
metadata_file,
Loader=yaml.SafeLoader,
) or {}
if package_specific_version.exists():
with package_specific_version.open(encoding="utf-8") as version_file:
yaml_specific_version = yaml.load(
version_file,
Loader=yaml.SafeLoader,
) or {}
yaml_base_info.update(yaml_specific_version)
self._normalize_vsix_metadata(yaml_base_info)
self.packages_info[settings.package_name] = yaml_base_info
else:
self.packages_info[settings.package_name] = dict()
self.logger.info("[LOAD METADATA]: not found description for package use default")
self.diff_exclude_files = []
self.diff_exclude_extensions = []
self.check_manual_processing()
@staticmethod
def _normalize_vsix_metadata(package_info: dict) -> None:
aliases = {
"install_dependencies": "install_actions",
"build_vsix": "build_actions",
"run_tests": "coverage_command",
"source_repository": "repository",
}
for source_key, target_key in aliases.items():
if source_key in package_info and target_key not in package_info:
package_info[target_key] = package_info[source_key]
coverage = package_info.get("coverage")
if isinstance(coverage, dict):
if "tool" in coverage and "coverage_tool" not in package_info:
package_info["coverage_tool"] = coverage["tool"]
if "enabled" in coverage and "coverage_enabled" not in package_info:
package_info["coverage_enabled"] = coverage["enabled"]
for value in package_info.values():
if isinstance(value, dict) and value is not coverage:
VSIX_T5Checker._normalize_vsix_metadata(value)
def check_manual_processing(self) -> None:
if self.get_version_specific_package_info(MANUAL_PROCESSING_KEY):
self.status.append_status_file(self.status.MANUAL_CHECK, '"Based on metadata info, manual check (T06) required"')
self.logger.info(MANUAL_CHECK_DETECTION_MESSAGE)
self.stage_warn_without_exit(
stage_name="manual_check_required",
details=f"Manual check is required to proceed with this package. \n{MANUAL_CHECK_DETECTION_MESSAGE}\n"
)
self.check_is_done()
def extract_package(self, archive_path: pathlib.Path, result_folder: pathlib.Path, substep_name: str) -> pathlib.Path:
try:
result_folder.mkdir(parents=True, exist_ok=False)
result_root = result_folder.resolve()
archive_source = archive_path
decompressed_path = None
with archive_path.open("rb") as package_file:
is_gzip_wrapped = package_file.read(2) == b"\x1f\x8b"
if is_gzip_wrapped:
decompressed_path = result_folder.parent / f"{archive_path.name}.decompressed"
with gzip.open(archive_path, "rb") as source, decompressed_path.open("wb") as output:
shutil.copyfileobj(source, output)
archive_source = decompressed_path
with zipfile.ZipFile(archive_source) as archive:
for member in archive.infolist():
destination = (result_folder / member.filename).resolve()
if destination != result_root and result_root not in destination.parents:
raise ValueError(f"Unsafe path in VSIX archive: {member.filename}")
archive.extractall(result_folder)
if decompressed_path:
decompressed_path.unlink()
except (OSError, ValueError, zipfile.BadZipFile) as e:
self.stage_error(
stage_name=substep_name,
details=f"Failed to extract VSIX archive {archive_path}: {e}.",
)
package_json = result_folder / "extension" / "package.json"
if not package_json.is_file():
self.stage_error(
stage_name=substep_name,
details=f"VSIX archive {archive_path} does not contain extension/package.json.",
)
self.stage_passed(stage_name=substep_name)
return result_folder
def load_extracted_package_json(self, package_extracted_path: pathlib.Path) -> None:
stage_name='load_extracted_package_json'
try:
with (package_extracted_path / "extension" / "package.json").open(
encoding="utf-8"
) as package_json:
self.extracted_package_info = json.load(package_json)
except (OSError, json.JSONDecodeError) as e:
self.stage_error(
stage_name=stage_name,
details=f"Failed to load extension/package.json from VSIX: {e}",
)
self.stage_passed(stage_name=stage_name)
def get_git_url(self):
try:
repo_url = self.get_version_specific_package_info("repository")
if isinstance(repo_url, dict):
repo_url = repo_url.get("url")
if not repo_url:
repository = self.extracted_package_info.get("repository")
if isinstance(repository, str):
repo_url = repository
elif isinstance(repository, dict):
repo_url = repository.get("url")
if not repo_url:
self.status.append_status_file(self.status.GIT_URL_ABSENT, '"Git url is absent in package info and metadata"')
self.stage_error(
stage_name="get_git_url",
details="Failed to fetch git url.",
)
repo_url = repo_url.removeprefix("git+")
if repo_url.startswith("git@github.com:"):
repo_url = "https://github.com/" + repo_url.removeprefix("git@github.com:")
parsed = urlparse(repo_url)
if parsed.hostname != "github.com":
self.status.append_status_file(self.status.GIT_URL_PARSE_FAILED, '"Failed to parse git url - not a github url"')
self.stage_error(
stage_name="get_git_url",
details=f"Failed to extract package repository - {repo_url}",
)
parts = [part for part in parsed.path.split("/") if part]
if len(parts) < 2:
raise ValueError(f"GitHub repository path is incomplete: {parsed.path}")
repo_url = "https://github.com/" + "/".join(parts[:2])
except Exception as e:
self.status.append_status_file(self.status.GIT_URL_EXTRACT_FAILED, '"Failed to extract git url - not a github url"')
self.stage_error(
stage_name="get_git_url",
details=f"Failed to extract package repository: {e}",
)
self.stage_passed(
stage_name="get_git_url",
details="git url successfully extracted"
)
self.logger.info("Git url for the project is: %s", repo_url)
return repo_url
def clone_package_and_checkout(self, working_dir: pathlib.Path) -> pathlib.Path:
stage_name = "clone_package_and_checkout"
repo_url = self.get_git_url()
project_dir = working_dir / "git_package" / self.package_name
project_dir.parent.mkdir(parents=True, exist_ok=True)
try:
subprocess.run(
["git", "clone", repo_url, str(project_dir)],
check=True,
text=True,
capture_output=True,
)
source_ref = self.get_version_specific_package_info("source_ref")
if source_ref:
subprocess.run(
["git", "fetch", "origin", str(source_ref)],
cwd=project_dir,
check=True,
text=True,
capture_output=True,
)
subprocess.run(
["git", "checkout", "--detach", "FETCH_HEAD"],
cwd=project_dir,
check=True,
text=True,
capture_output=True,
)
else:
commit_id = self.get_version_specific_package_info("commit_id")
if commit_id:
subprocess.run(
["git", "checkout", "--detach", str(commit_id)],
cwd=project_dir,
check=True,
text=True,
capture_output=True,
)
else:
tags = subprocess.run(
["git", "tag", "--list", self.version, f"v{self.version}"],
cwd=project_dir,
check=True,
text=True,
capture_output=True,
).stdout.splitlines()
if tags:
preferred_tag = self.version if self.version in tags else tags[0]
subprocess.run(
["git", "checkout", "--detach", preferred_tag],
cwd=project_dir,
check=True,
text=True,
capture_output=True,
)
else:
self.logger.warning(
"No tag matching %s or v%s; checking current repository HEAD",
self.version,
self.version,
)
subprocess.run(
["git", "submodule", "update", "--init", "--recursive"],
cwd=project_dir,
check=True,
text=True,
capture_output=True,
)
except (OSError, subprocess.CalledProcessError) as e:
stderr = getattr(e, "stderr", None)
self.stage_error(
stage_name=stage_name,
details=f"Failed to clone or checkout {repo_url}: {stderr or e}",
)
self.stage_passed(stage_name=stage_name, details="Finished downloading package sources")
return project_dir
def load_repo_package_info(self, package_dir):
package_file_path = self.get_version_specific_package_info('repo_package_json_path')
if not package_file_path:
package_file_path = self.get_package_info('repo_package_json_path')
if not package_file_path:
package_file_path = "package.json"
try:
with (package_dir / package_file_path).open(encoding="utf-8") as package_json:
res = json.load(package_json)
if 'scripts' in res:
from pprint import pformat
self.logger.debug("Package Scripts: %s", pformat(res['scripts']))
except json.JSONDecodeError as e:
self.stage_error(
stage_name="load_repo_package_info",
details=f"Failed to load repo package info - failed to decode {package_file_path} json {e.msg}",
)
except OSError as e:
self.stage_error(
stage_name="load_repo_package_info",
details=f"Failed to load repo package info - failed to open {package_file_path}: {e}",
)
self.stage_passed(stage_name="load_repo_package_info")
return res
def install_package(
self,
*,
package_dir: pathlib.Path,
env: dict,
**kwargs,
) -> None:
actions = self.get_actions_from_config("install_actions")
if actions is None:
self.install_package_general(package_dir=package_dir, env=env)
else:
self.run_actions(
"install_package",
actions,
env=env,
package_dir=package_dir,
check=True,
timeout=self.install_timeout,
)
self.stage_passed(stage_name="install_package", details="Dependencies were installed.")
def install_package_general(self, package_dir: pathlib.Path, env: dict) -> bool:
try:
if (package_dir / "pnpm-lock.yaml").is_file():
actions = [
"corepack enable",
"pnpm install --frozen-lockfile",
]
elif (package_dir / "yarn.lock").is_file():
actions = [
"corepack enable",
"yarn install --frozen-lockfile --non-interactive",
]
elif (package_dir / "package-lock.json").is_file():
actions = ["npm ci"]
else:
actions = ["npm install --no-audit"]
return self.run_actions(
"install_package_general",
actions,
env=env,
package_dir=package_dir,
check=True,
timeout=self.install_timeout,
)
except Exception as e:
self.stage_error(
stage_name="install_package_general",
details=f"Failed to install source dependencies: {e}",
)
def custom_prepare(self, package_dir: str | pathlib.Path, env: dict) -> bool:
"""Run prepare actions from yaml file"""
self.logger.info("Run custom package preparation")
prepare = self.get_actions_from_config('prepare')
result = self.run_actions(
'prepare',
prepare,
env=env,
package_dir=package_dir,
check=True,
timeout=self.install_timeout
)
return result
def build_package(
self,
*,
package_dir: pathlib.Path,
env: dict,
**kwargs,
) -> bool:
actions = self.get_actions_from_config("build_actions")
if actions is None:
result = self.build_package_general(package_dir=package_dir, env=env)
else:
result = self.run_actions(
"build_package",
actions,
env=env,
package_dir=package_dir,
check=True,
timeout=self.build_timeout,
)
self.stage_passed(stage_name="build_package", details="VSIX was built successfully.")
return result
def build_package_general(self, env: dict, package_dir: str | pathlib.Path) -> bool:
return self.run_actions(
'build_package_general',
["npx @vscode/vsce package --out source.vsix"],
env=env,
package_dir=package_dir,
check=True,
timeout=self.build_timeout
)
def find_built_vsix(
self,
package_dir: pathlib.Path,
existing_archives: dict[pathlib.Path, int],
) -> pathlib.Path:
configured_path = self.get_version_specific_package_info("built_vsix_path")
if configured_path:
built_vsix = package_dir / configured_path
if built_vsix.is_file():
return built_vsix
self.stage_error(
stage_name="find_built_vsix",
details=f"Configured built VSIX was not found: {built_vsix}",
)
candidates = []
for archive in package_dir.rglob("*.vsix"):
mtime = archive.stat().st_mtime_ns
if archive not in existing_archives or existing_archives[archive] != mtime:
candidates.append(archive)
if not candidates:
self.stage_error(
stage_name="find_built_vsix",
details="Build command did not create or update a VSIX archive.",
)
built_vsix = max(candidates, key=lambda path: path.stat().st_mtime_ns)
self.stage_passed(
stage_name="find_built_vsix",
details=f"Built VSIX found at {built_vsix}",
)
return built_vsix
def count_coverage_custom(self, env: dict, package_dir: str | pathlib.Path) -> None:
coverage_actions = self.get_actions_from_config("coverage_command")
if not coverage_actions:
# No actions provided. This is weird but who to judge?
return
self.logger.info("Run custom package coverage")
self.run_actions_with_monitor(
'count_coverage_custom',
coverage_actions,
env=env,
check=False,
package_dir=package_dir,
timeout=self.test_timeout
)
return
def count_coverage_general(self, env: dict, package_dir: str | pathlib.Path, **kwargs) -> None:
self.status.append_status_file(self.status.COVERAGE_COMMAND_ABSENT, '"TEST_COMMAND section in metadate must be filled"')
self.stage_error(
stage_name='count_coverage_general',
details="Coverage command for the VSIX source repository is not configured."
)
def run_coverage_report(self, package_dir: pathlib.Path, env: dict) -> str:
coverage_tool_name = self.get_version_specific_package_info('coverage_tool')
if coverage_tool_name not in self.supported_coverage_tools:
self.stage_error(
stage_name="run_coverage_report",
details=f"Unsupported coverage tool: `{coverage_tool_name}`"
)
coverage_tool = self.supported_coverage_tools[coverage_tool_name]
coverage_report_folder = coverage_tool['coverage_report_folder']
# Will travers from 'build_path' up to 'package_path' in hope to find coverage dir.
build_relative_path = self.get_version_specific_package_info('build_relative_path')
if build_relative_path:
probe_path = package_dir / build_relative_path
while probe_path != package_dir:
if (probe_path / coverage_report_folder).exists():
package_dir = probe_path
break
probe_path = probe_path.parent
if not (package_dir / coverage_report_folder).exists():
self.stage_error(
stage_name="run_coverage_report",
details=f"Failed to find coverage report dir."
)
# TODO: capture report and save in metadata
coverage_report = self.run_actions(
'run_coverage_report',
coverage_tool['report'],
env=env,
check=True,
package_dir=package_dir,
)
self.stage_passed('run_coverage_report', details=coverage_report)
coverage_report = self.run_actions(
'process_coverage_result',
coverage_tool['check'],
env=env,
check=True,
package_dir=package_dir,
)
self.stage_passed('process_coverage_result', details=coverage_report)
def run_tests_coverage(self, env: dict, package_dir: str | pathlib.Path, **kwargs) -> None:
if self.get_version_specific_package_info("coverage_enabled") is False:
self.stage_passed(
stage_name="count_coverage",
details="Coverage verification is disabled for this VSIX package.",
)
return
coverage_actions = self.get_actions_from_config("coverage_command")
if coverage_actions is None:
self.count_coverage_general(env=env, package_dir=package_dir, **kwargs)
else:
self.count_coverage_custom(env, package_dir)
self.run_coverage_report(package_dir, env)
self.stage_passed(
stage_name="count_coverage",
details=f"Package {self.package_name} {self.version} was processed."
)
@staticmethod
def get_package_env():
env = os.environ.copy()
new_env_path = []
for path in env['PATH'].split(os.pathsep):
if path.endswith('venv/bin'):
# erase current venv
continue
new_env_path.append(path)
env['PATH'] = os.pathsep.join(new_env_path)
env.pop('PYTHONHOME', None)
return env
def run_check(self):
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_path = pathlib.Path(tmpdir)
marketplace_extracted_path = self.extract_package(
archive_path=self.package_path,
result_folder=tmpdir_path / "marketplace_vsix",
substep_name="extract_marketplace_vsix",
)
self.load_extracted_package_json(marketplace_extracted_path)
self.logger.info("Working on VSIX package %s@%s", self.package_name, self.version)
git_project_path = self.clone_package_and_checkout(
working_dir=pathlib.Path(self.report_dir)
)
self.repo_package_info = self.load_repo_package_info(git_project_path)
package_env = self.get_package_env()
self.custom_prepare(package_dir=git_project_path, env=package_env)
self.install_package(package_dir=git_project_path, env=package_env)
existing_archives = {
archive: archive.stat().st_mtime_ns
for archive in git_project_path.rglob("*.vsix")
}
self.build_package(package_dir=git_project_path, env=package_env)
built_vsix = self.find_built_vsix(git_project_path, existing_archives)
source_extracted_path = self.extract_package(
archive_path=built_vsix,
result_folder=tmpdir_path / "source_vsix",
substep_name="extract_source_vsix",
)
self.check_diff(marketplace_extracted_path, source_extracted_path)
self.run_tests_coverage(package_dir=git_project_path, env=package_env)
self.check_is_done()
def main(args):
report_folder = args.report_path
package_file_name = settings.package_file
logger = get_logger(
"vsix_package_processing",
log_file=pathlib.Path(report_folder) / f"processing_{package_file_name}.log",
log_level_name=settings.log_level.upper()
)
logger.info("Start processing %s", package_file_name)
scanfeeder = Scanfeeder(logger)
monitor = Monitor(
tcpdump_enable=settings.monitor_enable,
sysdig_enable=settings.monitor_enable,
scanfeeder=scanfeeder,
logger=logger,
pipe_path=settings.sysdig_pipe_path,
report_folder=report_folder,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
vsix_checker = VSIX_T5Checker(
scanfeeder=scanfeeder,
monitor=monitor,
logger=logger,
report_dir=pathlib.Path(args.report_path),
package_path=pathlib.Path(args.file_path),
)
vsix_checker.run_check()
if __name__ == "__main__":
projdir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
parser = argparse.ArgumentParser(prog="T005 checker", description="does t005 check")
parser.add_argument("--file_path", help="full path to the archive to be check", required=False, default=os.path.join(projdir, 'input', settings.package_file))
parser.add_argument("--report_path", help="full path to the directory where report will be generated", required=False, default=os.path.join(projdir, 'report'))
args = parser.parse_args()
main(args)
+3
View File
@@ -0,0 +1,3 @@
Host github.com
User git
IdentityFile ~/.ssh/github_rsa
+38
View File
@@ -0,0 +1,38 @@
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlgAAAAdzc2gtcn
NhAAAAAwEAAQAAAYBqeOy6x16OhobcdREnOAJT0/0mpSS9svpXuFX/yiFCAez0SOW2xVdV
VDjPIyz8itFhExxtJmZmjp+uIQU4fYarzkEnQBHGmtmQJLGvRC04CvjoKLzjmkyDV8Ym9b
mY4hh8fdS3wMHjg+JaKWAN9f334gpJaaJ6uBvDsFGC41GYN2EWXd1cig3EY8IQzzqZjC0h
yWc7QHpNjzPHjH/IdPXzBEjIEhSN0+s/SCKX50cCJBcCu17o6nUuajSQEV+FbCOExWsvCV
p/Nd298eiHsQhB71ZrLUtsWu3tH4Fh5PglLovbret3zPzv1GqfC03QJu3Z5X011QiGF2S6
H4gRjC9R5nNP//OuPcNvJm3fXFwXAQ6xZ5xUTm4cGCu3ICRpOJX73iy0CR3iFdCI6SvlBO
9CzKAoRdcfXlnPvA1usOysY6al7Mtf7OdQ+ZTAI2faej6QmQ3qIFR9HiNqns2ytdHQU6h/
SlGb6xpw7gvUEAHx4b707Znv2JCcvMyvo5EP6IUAAAWQizrn0Ys659EAAAAHc3NoLXJzYQ
AAAYBqeOy6x16OhobcdREnOAJT0/0mpSS9svpXuFX/yiFCAez0SOW2xVdVVDjPIyz8itFh
ExxtJmZmjp+uIQU4fYarzkEnQBHGmtmQJLGvRC04CvjoKLzjmkyDV8Ym9bmY4hh8fdS3wM
Hjg+JaKWAN9f334gpJaaJ6uBvDsFGC41GYN2EWXd1cig3EY8IQzzqZjC0hyWc7QHpNjzPH
jH/IdPXzBEjIEhSN0+s/SCKX50cCJBcCu17o6nUuajSQEV+FbCOExWsvCVp/Nd298eiHsQ
hB71ZrLUtsWu3tH4Fh5PglLovbret3zPzv1GqfC03QJu3Z5X011QiGF2S6H4gRjC9R5nNP
//OuPcNvJm3fXFwXAQ6xZ5xUTm4cGCu3ICRpOJX73iy0CR3iFdCI6SvlBO9CzKAoRdcfXl
nPvA1usOysY6al7Mtf7OdQ+ZTAI2faej6QmQ3qIFR9HiNqns2ytdHQU6h/SlGb6xpw7gvU
EAHx4b707Znv2JCcvMyvo5EP6IUAAAADAQABAAABgEztV+VPnQMPMathLeaPMidNGC/uy8
//2567ge9RJJxjdce+T7KxfHsbYfxPJvbEealQvI4kW4In9DlEhFgfYckKOcrZIL7FLQJX
cMX4VU+TpelfXbJ4zmVrarDFg7ZxTU3NQH8PV0dCB64rDMqpfDi4acRG/Wz7Tpwh7ZLLGS
4IfS5C3Wg6Gt5VUWcBww4j2Iv7z4lg+bzGBZxc6Ap2bymJY+iiH5Wlkufi/5OPN5/VYYbI
yDczBz1k4Ntioghta3hRBE8z2uu2hrPHlh3JKaX1MTsgN2UsP/Bak27yXEqvRL6ec0lejf
z4PcyzPNhTkgJXCWfnc0ARivOXvp+yUrAmrfhB78Mjh1SN4xwvpzjnVDsH1+M7TYxj8vFx
inGcSFRSkKCd04n8GUat298EADrRsXDwPaZNSLLsAypVRAHt7eA46n/5m3pCUNvFWvRQl8
wzDJPwDw331Q6auFfDPiO/v5a7z8wCbEyFCiPRjkQZWGpL7EZaB8OvQGnCjbhZ9QAAAMAx
MamawSxfa+ggd4qJrunm9nuowE00VktGsEQRQ6S4cT6B5A/+Y+UyJvf6M+FdqNdpgh7sCr
dvwV1I8NZ3++zjtb6vdY7sc60K8OwFpj8OrAaUyK76jd6TPy45mIqpZqR8bIspt511bSdW
1rwHcYqbrifCiatSNuniArKeLwz38mYUeLy/eJ+5wZJaBVCipjrZWHVTSwnCp9FzBueCQ0
cDq/V1jI+/4EJiTgvsD470U4FB09C1f/hbanXluxgUYRMAAADBANpgJi/jQ3m5Ye9/UXvV
sQNlDp011JxMfYSqIUsdSYXUYi7yA4Jfh+P1xi9wiyD4VqseXeFo6RcVnt99NyLfhUvyrX
dVYqS6FvVK0DePWbdeW40DaLulx5PLeWgjaSut/n1opGIyVp/34xd4ZLTQwowv64NoJIi3
cuRW4rxNK6YxkzYAsRcu9BPH003JPQcy06n7+tUGQDVqAJnHUHsx5ZPBXfxxmfPY/B+hn7
SrINRRS1PaKTrk6sY/TJn8qoUPlwAAAMB80RTKrxJxy9vOixp1XgMN/yUZVHUUn2l35B3Q
aTbXu5yGMiaTtQfydrBNqwwQEYSy2I3ZMFv3dgQ3bSHDo7mpV+0lY7RO7m6ZU9nv0jlp2G
axeMztgZ4E2a2ioKZPr9TkPb5o+d7kDAO8LEUrb59aMIhGgPwj24c63auYWvjcXGjLCx2P
oAoQ0ytioRA2Ers1iDe89RVCLcv8ciKQiNOzx0s9BpUs3ifRJeR+Ah6dkxDLz9y3DwD0Ob
xOiIDETEMAAAAba3VyZ3Vza2lub3NAbWFjLWt1cmd1c2tpbm9zAQI=
-----END OPENSSH PRIVATE KEY-----
+1
View File
@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgGp47LrHXo6Ghtx1ESc4AlPT/SalJL2y+le4Vf/KIUIB7PRI5bbFV1VUOM8jLPyK0WETHG0mZmaOn64hBTh9hqvOQSdAEcaa2ZAksa9ELTgK+OgovOOaTINXxib1uZjiGHx91LfAweOD4lopYA31/ffiCklponq4G8OwUYLjUZg3YRZd3VyKDcRjwhDPOpmMLSHJZztAek2PM8eMf8h09fMESMgSFI3T6z9IIpfnRwIkFwK7XujqdS5qNJARX4VsI4TFay8JWn813b3x6IexCEHvVmstS2xa7e0fgWHk+CUui9ut63fM/O/Uap8LTdAm7dnlfTXVCIYXZLofiBGML1Hmc0//8649w28mbd9cXBcBDrFnnFRObhwYK7cgJGk4lfveLLQJHeIV0IjpK+UE70LMoChF1x9eWc+8DW6w7KxjpqXsy1/s51D5lMAjZ9p6PpCZDeogVH0eI2qezbK10dBTqH9KUZvrGnDuC9QQAfHhvvTtme/YkJy8zK+jkQ/ohQ== x@x
+3
View File
@@ -0,0 +1,3 @@
github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=