diff --git a/Makefile b/Makefile index 57176c5..986a925 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,7 @@ -ifneq (, $(shell ls ../../vision-guard-share/ | grep headers.mk)) -include ../../vision-guard-share/headers.mk -export MAIN_TRUNC=yes -GET_VERSION_DEP=get_version -else export MAIN_TRUNC=no LOCAL_BUILD=1 FINAL_VERSION=latest GET_VERSION_DEP= -endif .PHONY: test test_coverage test_ci lint lint_ci dep fmt build build_scanfeeder build_ci apply_git_hooks postclone precommit clean build_docker_image docker_image_ci help get_version get_build_date diff --git a/apt-requirements.txt b/apt-requirements.txt new file mode 100644 index 0000000..1146bb8 --- /dev/null +++ b/apt-requirements.txt @@ -0,0 +1,74 @@ +tzdata +bash +git +python3-dev +python3-full +python3-pip +python3-virtualenv +python3-venv +python3-pybind11 +python3-setuptools +build-essential +automake +autoconf +ninja-build +pandoc +cmake +wget +bc +curl +diffutils +libcurl4-openssl-dev +libopenblas-dev +libomp-dev +libsndfile1 +ffmpeg +cpio +gcc-12 +gfortran +gnupg +unzip +libtiff5-dev +libjpeg8-dev +libopenjp2-7-dev +zlib1g-dev +libfreetype6-dev +liblcms2-dev +libwebp-dev +tcl8.6-dev +tk8.6-dev +libharfbuzz-dev +libfribidi-dev +libxcb1-dev +meson +rsync +llvm +inkscape +xvfb +x11-utils +libxkbcommon-x11-0 +libxcb-xinerama0 +python3-pyqt5 +abcm2ps +graphviz +graphviz-dev +default-jre +software-properties-common +libzmq3-dev +libsodium-dev +libsasl2-dev +libmagic-dev +libffi-dev +libncurses5-dev +libncursesw5-dev +zlib1g +libbz2-dev +libsqlite3-dev +libssl-dev +libreadline-dev +vim +kmod +ubuntu-dbgsym-keyring +tcpdump +libcap2-bin +openjdk-17-jdk diff --git a/check.sh b/check.sh new file mode 100755 index 0000000..17b5c80 --- /dev/null +++ b/check.sh @@ -0,0 +1,41 @@ +#!/bin/bash +mkdir input report output +export NPM_RUNTIME_CHECK_VER=latest +PACKAGE_NAME=${1} +PACKAGE_VERSION=${2} + + +RESULT_NAME=$PACKAGE_NAME + +if [ ${PACKAGE_NAME:0:1} == '@' ] +then + POS=$(echo $PACKAGE_NAME | grep -ob "/" | grep -oE "[0-9]+" | head -n 1) + RESULT_NAME=${PACKAGE_NAME:$POS+1} +fi + +URL=$(npm view ${PACKAGE_NAME}@${PACKAGE_VERSION} dist.tarball) + +REPORT=$(ls ./report/git_package) +REPORT_LEN=${#REPORT} +if [ ${REPORT_LEN} != 0 ] +then + sync + rm -rf ./report/git_package + echo "Clean git_package" +fi + +RESULT_INPUT=${RESULT_NAME}-${PACKAGE_VERSION}.tgz + +cd input + if [ $(ls $RESULT_INPUT) ] + then + rm $RESULT_INPUT + fi + set -x + echo $URL + wget -O "${RESULT_INPUT}" "${URL}" +cd .. + +export RELOAD_CONFIG=1 +export ADD_METADATA_VOLUME='true' +bash ./run.sh "${RESULT_INPUT}" "$PACKAGE_NAME" "$PACKAGE_VERSION" diff --git a/pipe_create.sh b/pipe_create.sh new file mode 100755 index 0000000..9f66748 --- /dev/null +++ b/pipe_create.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +PIPE_PATH=$1 + +if [ "${PIPE_PATH}" == "" ]; then + echo "There is no defined path to pipe" + + exit 1 +fi + +rm ${PIPE_PATH} +mkfifo ${PIPE_PATH} + +if [ $? -ne 0 ]; then + echo "Can't create pipe" + + exit 1 +fi \ No newline at end of file diff --git a/pipe_listener.sh b/pipe_listener.sh new file mode 100755 index 0000000..6b314fb --- /dev/null +++ b/pipe_listener.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +PIPE_FILE=$1 + +if [ "${PIPE_FILE}" == "" ]; then + echo "Pipe is not presented" + + exit 1 +fi + +if ! [ -p ${PIPE_FILE} ]; then + echo "There is no such pipe here" + + exit 1 +fi + +PID= +CONTAINER_ID= +VPID= +while true; +do + STR=$(cat ${PIPE_FILE}) + + if [ $? -ne 0 ]; then + echo "Seems that pipe has gone and no one holds me. Preparing to die..." + + exit 1 + fi + + echo "Caught through pipe: ${STR}" + + grep sysdig_start <<< ${STR} + + if [ $? -eq 0 ]; then + if [ "${PID}" != "" ]; then + echo "You asking to start sysdig, but we already started process ${PID}. So, ignoring" + continue + fi + + CONTAINER_ID=$(docker ps -a | grep npm_runtime_check | awk '{print $1;}') + SYSDIG_FILE=$(awk '{print $3}' <<< ${STR}) + + if [ "${CONTAINER_ID}" == "" ]; then + echo "sysdig_start caught but no container id was presented" + else + docker inspect ${CONTAINER_ID} > /dev/null + if [ $? -ne 0 ]; then + echo "Can't inspect ${CONTAINER_ID} container. No sysdig will be called." + else + sudo sysdig container.id=${CONTAINER_ID} and container.name!=host -w ${REPORT_DIR}/${SYSDIG_FILE} -z & + PID=$! + echo "sysdig started with pid ${PID} to "${REPORT_DIR}/${SYSDIG_FILE} + sleep 2 + sudo chown ${DST_USER_ID}:${DST_GROUP_ID} -R ${REPORT_DIR}/${SYSDIG_FILE} + fi + fi + echo -n "sysdig_started" > ${PIPE_FILE} + else + grep sysdig_pid <<< ${STR} + + if [ $? -eq 0 ]; then + VPID=$(awk '{print $2}' <<< ${STR}) + echo "vpid is "${VPID} + echo sysdig_pidded > ${PIPE_FILE} + fi + fi + + case ${STR} in + + sysdig_stop) + if [ "${PID}" != "" ]; then + echo "sysdig_stop caught. Perform: kill -INT ${PID}" + eval "kill -INT ${PID}" + wait ${PID} + HPID=$(sudo sysdig -r ${REPORT_DIR}/${SYSDIG_FILE} proc.vpid=${VPID} -p '%proc.pid' -n 1) + echo -n ${HPID} > ${REPORT_DIR}/${SYSDIG_FILE}.pid + + PID= + CONTAINER_ID= + VPID= + HPID= + echo -n "sysdig_stopped" > ${PIPE_FILE} + else + echo "PID is not presented, but sysdig_stop was called. Ignoring." + echo -n "sysdig_stopped" > ${PIPE_FILE} + fi + ;; + + kill) + echo "Someone asked me to die. Doing so" + exit 0 + ;; + esac +done \ No newline at end of file diff --git a/pipe_remove.sh b/pipe_remove.sh new file mode 100755 index 0000000..670c69e --- /dev/null +++ b/pipe_remove.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +PIPE_PATH=$1 + +if [ "${PIPE_PATH}" == "" ]; then + echo "There is no defined path to pipe" + + exit 1 +fi + +echo -n sysdig_stop > ${PIPE_PATH} +cat ${PIPE_PATH} +echo -n kill > ${PIPE_PATH} +rm ${PIPE_PATH} + +if [ $? -ne 0 ]; then + echo "Can't remove pipe" + + exit 1 +fi \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..6297595 --- /dev/null +++ b/run.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +finish_with_pipe() { + if [ "${USE_SYSDIG}" == "true" ]; then + ${SCRIPT_DIR}/pipe_remove.sh ${SYSDIG_PIPE_PATH} + + RES=$? + + ps aux | grep 'sysdig\|pipe_listener.sh' | awk '{ print $2; }' | xargs kill -INT + + if [ ${RES} != 0 ]; then + exit 1 + fi + fi +} + +# files package +PACKAGE_FILE=${1} +# name package +PACKAGE_NAME=${2} +# version package +PACKAGE_VERSION=${3} +# input dir from $4 or default ./input +INPUT_DIR=${4:-./input} +# report dir from $5 or default ./report +REPORT_DIR=${5-./report} +# id of the pipeline, see result of "/pipeline/start" +PIPELINE_ID=${6} +# base url of the scanhub api, default http://localhost:3000/api/v1 +SCANHUB_URL=${7:-"http://localhost:3000/api/v1"} +# output dir, default ./status +STATUS_DIR=${8:-./status} +# MinIO api url, default 127.0.0.1:9000 +MINIO_API_URL=${9:-"127.0.0.1:9001"} # для локального теста на 127.0.0.1, нужно выставить NETWORK=host +# MinIO user, default is admin +MINIO_USER=${10:-"admin"} +# MinIO password, default is password +MINIO_PASSWORD=${11:-"password"} +# MinIO bucket name, default is +MINIO_BUCKET=${12:-"taskalo-files"} +PLUGIN_NAME="npm_runtime_check" +SCANFEEDER_DEBUG=${SCANFEEDER_DEBUG:-true} +USE_SYSDIG=${USE_SYSDIG:-false} +DST_USER_ID=$(id -u) +DST_GROUP_ID=$(id -g) + + +NETWORK=${NETWORK:-default} # для локального теста на 127.0.0.1, нужно выставить NETWORK=host +SCRIPT_DIR=$(realpath "$( dirname -- "${BASH_SOURCE[0]}" )") +SYSDIG_PIPE_PATH=${SYSDIG_PIPE_PATH:-${SCRIPT_DIR}/sysdig_pipe} +LISTENER_LOG_FILE=${REPORT_DIR}/pipe_listener.log + +if [ "${USE_SYSDIG}" == "true" ]; then + sysdig=$(which sysdig) + + if [ $? != 0 ]; then + echo "sysdig is not presented in the system, please install it." + exit 1 + fi + + ${SCRIPT_DIR}/pipe_create.sh ${SYSDIG_PIPE_PATH} + if [ $? != 0 ]; then + echo "Can't create pipe for communication" + exit 1 + fi + + REPORT_DIR=${REPORT_DIR} ${SCRIPT_DIR}/pipe_listener.sh ${SYSDIG_PIPE_PATH} ${DST_USER_ID} ${DST_GROUP_ID} 2>&1 > ${LISTENER_LOG_FILE} & +fi + +MONITOR_ENABLE=$([ "${USE_SYSDIG}" = "true" ] && echo "True" || echo "False") +USER_HOME='/home/nandy' +ADD_METADATA_VOLUME=$([ "${ADD_METADATA_VOLUME}" = "true" ] && echo "-v ${PWD}/metadata:${USER_HOME}/metadata" || \ + echo "") + + +NPM_RUNTIME_CHECK_IMAGE=${NPM_RUNTIME_CHECK_IMAGE:-npm_runtime_check} + +echo "=====================> start ${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest}" + +docker run --rm -i \ + --network ${NETWORK} \ + --add-host=host.docker.internal:host-gateway \ + --privileged \ + --cap-add=NET_ADMIN \ + --cap-add=SYS_ADMIN \ + ${ADD_METADATA_VOLUME} \ + -v $INPUT_DIR:/input:ro \ + -v $REPORT_DIR:/report \ + -v $STATUS_DIR:/status \ + -v ${SYSDIG_PIPE_PATH}:/pipe \ + -e REPORT_DIR=/report/ \ + -e PACKAGE_FILE=$PACKAGE_FILE \ + -e PACKAGE_NAME=$PACKAGE_NAME \ + -e PACKAGE_VERSION=$PACKAGE_VERSION \ + -e METADATA_PATH="${USER_HOME}/metadata" \ + -e PLUGIN_NAME="${PLUGIN_NAME}" \ + -e PIPELINE_ID=${PIPELINE_ID} \ + -e SCANHUB_URL=${SCANHUB_URL} \ + -e MINIO_API_URL="$MINIO_API_URL" \ + -e MINIO_USER="$MINIO_USER" \ + -e MINIO_PASSWORD="$MINIO_PASSWORD" \ + -e MINIO_BUCKET="$MINIO_BUCKET" \ + -e DST_USER_ID=${DST_USER_ID} \ + -e DST_GROUP_ID=${DST_GROUP_ID} \ + -e SYSDIG_PIPE_PATH=/pipe \ + -e SCANFEEDER_DEBUG=${SCANFEEDER_DEBUG} \ + -e MONITOR_ENABLE=${MONITOR_ENABLE} \ + -e USER_HOME=${USER_HOME} \ + -e LOG_LEVEL="DEBUG" \ + -e T05_DEFAULT_TIMEOUT=1800 \ + -e SCANFEEDER_PATH="/usr/local/bin/scanfeeder" \ + "${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest}" bash -l -c "source ${USER_HOME}/processing/venv/bin/activate && python3 ${USER_HOME}/processing/npm_package_processing.py --file_path=/input/${PACKAGE_FILE} --report_path=/report" + +RES_PLUGIN=$? +echo "=====================> finished ${NPM_RUNTIME_CHECK_IMAGE}:${NPM_RUNTIME_CHECK_VER:-latest} with code ${RES_PLUGIN}" + +finish_with_pipe +echo "Npm runtime check is done, exit with code ${RES_PLUGIN}" +exit ${RES_PLUGIN}