first commit

tests/check/test_content.py

@@ -0,0 +1,25 @@
+import re
+import os
+
+FILE_EXCLUDE_PATTERN = re.compile(r'[/\\]\.(git|idea|vscode|pytest_cache)[/\\]')
+
+
+def is_textplain(data: bytes):
+    return b'\x00' not in data
+
+
+def test_content():
+    basedir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..'))
+    for (now_dir, dirs, files) in os.walk(basedir):
+        for name in files:
+            filename = os.path.join(now_dir, name)
+            if FILE_EXCLUDE_PATTERN.search(filename):
+                continue
+
+            with open(filename, 'rb') as f:
+                data = f.read()
+
+            if not is_textplain(data):
+                continue
+
+            assert b'\r\n' not in data, f'CRLF must be convert to LF for Vulhub files, but {filename} did not'

tests/check/test_dockerfile.py

@@ -0,0 +1,18 @@
+import os
+import subprocess
+
+
+def test_dockerfile_lint():
+    basedir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..'))
+
+    dockerfiles = []
+    for (now_dir, dirs, files) in os.walk(basedir):
+        for name in files:
+            if name in ('oracle-java', ):
+                continue
+
+            if name == 'Dockerfile':
+                dockerfiles.append(os.path.join(now_dir, name))
+
+    config = os.path.join(basedir, 'tests', 'hadolint.yaml')
+    subprocess.run(['hadolint', '--config', config, '--failure-threshold', 'error'] + dockerfiles, check=True)

tests/check/test_env_toml.py

@@ -0,0 +1,50 @@
+import os
+import glob
+import tomllib
+import difflib
+
+basedir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..'))
+
+
+def test_toml_format():
+    with open(os.path.join(basedir, 'environments.toml'), 'rb') as f:
+        data = tomllib.load(f)
+
+    for env in data['environment']:
+        assert 'name' in env
+        assert 'cve' in env
+        assert 'app' in env
+        assert 'path' in env
+        assert 'tags' in env
+        assert len(env) == 5
+
+        assert len(env['tags']) > 0
+        assert isinstance(env['name'], str)
+        assert isinstance(env['cve'], list)
+        assert isinstance(env['app'], str)
+        assert isinstance(env['path'], str)
+        assert isinstance(env['tags'], list)
+        assert os.path.exists(os.path.join(basedir, env['path']))
+
+        blocks = env['path'].split('/')
+        assert len(blocks) == 2
+
+        assert len(data['tags']) > 0
+        for tag in env['tags']:
+            assert tag in data['tags']
+
+
+def test_environments_files():
+    with open(os.path.join(basedir, 'environments.toml'), 'rb') as f:
+        data = tomllib.load(f)
+
+    compose_files = [name.replace('\\', '/') for name in sorted(glob.glob("**/docker-compose.yml", recursive=True))]
+    env_files = []
+    for env in data['environment']:
+        files = os.listdir(os.path.join(basedir, env['path']))
+        assert 'README.md' in files, f"README.md not found in {env['path']}"
+        assert 'README.zh-cn.md' in files, f"README.zh-cn.md not found in {env['path']}"
+        assert 'docker-compose.yml' in files, f"docker-compose.yml not found in {env['path']}"
+        env_files.append(env['path'] + "/docker-compose.yml")
+
+    assert len(compose_files) == len(env_files), f"Do not forget to add new environment in environments.toml, difference: \n{'\n'.join(difflib.unified_diff(compose_files, env_files))}"

tests/check/test_filename.py

@@ -0,0 +1,48 @@
+import os
+import re
+
+
+basedir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..'))
+ARCHIVE_FILE_PATTERN = re.compile(r'^.*\.(tar\.gz|zip|7z|rar|exe|jar|xz|gz|tar|war)$', re.I)
+ARCHIVE_EXCEPTED = re.compile(r'[/\\](struts2|weblogic[/\\]weak_password)[/\\]')
+
+
+def test_dir_islower():
+    for name in os.listdir(basedir) + os.listdir(os.path.join(basedir, 'base')):
+        if not os.path.isdir(name):
+            continue
+
+        assert name.islower()
+
+
+def test_filename_format():
+    """
+    We are not allowed uppercase software directory name
+    """    
+    for (root, _, files) in os.walk(basedir):
+        if os.path.basename(root).startswith('.'):
+            continue
+
+        for name in files:
+            # check if extension is lowercase
+            fullname = os.path.join(root, name)
+            _, ext = os.path.splitext(name)
+            assert ext == ext.lower(), 'file extension must be lowercase, not %r' % name
+
+            # check if docker-compose.yaml is used
+            assert name != "docker-compose.yaml", "docker-compose.yaml is not allowed, use docker-compose.yml instead"
+
+            # check if readme file name is correct
+            if name.lower() == 'readme.md':
+                assert name == 'README.md', "README filename must be 'README.md', not %r" % name
+
+            # check if readme.zh-cn.md file name is correct
+            if name.lower() == 'readme.zh-cn.md':
+                assert name == 'README.zh-cn.md', "README.zh-cn filename must be 'README.zh-cn.md', not %r" % name
+
+            if os.path.isdir(fullname) and (name.lower().startswith('cve-') or name.lower().startswith('cnvd-') or name.lower().startswith('cnnvd-')):
+                assert name == name.upper(), "CVE/CNVD/CNNVD directory name must be uppercase, not %r" % name
+
+            # check if archive file size is lower than 4096 bytes
+            if ARCHIVE_FILE_PATTERN.match(name) is not None and ARCHIVE_EXCEPTED.search(fullname) is None:
+                assert os.path.getsize(fullname) <= 4096, "You should not upload a archive file larger than 4096 bytes"

tests/cleanup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -ex
+
+cleanupImages() {
+    docker images | grep 'auto-' | awk '{print $3}' | xargs docker rmi 
+}
+
+cleanupImages || echo "some images is not cleanup"
+
+echo "finish images cleanup"
+
+exit 0

tests/hadolint.yaml

@@ -0,0 +1,13 @@
+ignored:
+  - DL3002
+  - DL3007
+  - DL3008
+override:
+  error: 
+    - DL3006
+    - DL3027
+    - DL3030
+    - DL3014
+    - DL3017
+    - DL3021
+    - DL3022

tests/image-build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+image_path="${1%%/}"
+image_name="$2"
+
+if [[ ! "$image_name" =~ ":" ]]; then 
+    image_name=${image_name}:$(date +%s)
+fi
+
+echo "Preparing test image $image_name"
+
+cd "$image_path" || exit 1
+docker build -t "$image_name" .

tests/images-build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+runTest() {
+    local image_path="$1"
+    local image_name="$2"
+    if [[ ! -d $image_path ]]; then
+        echo "error message: image path \"$image_path\" not exists"
+        exit 1
+    fi
+
+    tag="auto-$(date +%s)"
+    cd "$image_path"
+    docker build -t "$image_name:$tag" .
+    cd "$OLDPWD"
+}
+
+for path in "$@"; do
+    image_path="${path%%/}"
+    image_name="${image_path//[\/\-.]/_}"
+    runTest "$image_path" "$image_name"
+    #echo "Image Name $image_name"
+done

tests/markdownlint.json

@@ -0,0 +1,10 @@
+{
+  "default": true,
+  "ul-indent": false,
+  "line_length": false,
+  "no-bare-urls": false,
+  "no-alt-text": false,
+  "fenced-code-language": false,
+  "no-inline-html": false,
+  "descriptive-link-text": false
+}

tests/tools/update_dockerhub.py

@@ -0,0 +1,123 @@
+import requests
+import os
+import sys
+import pathlib
+import logging
+import yaml
+from typing import Mapping, Iterable
+from collections import defaultdict
+
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+auth_token = os.environ.get('TOKEN', '')
+session = requests.session()
+session.headers = {
+    'Authorization': f'Bearer {auth_token}',
+    'Origin': 'https://hub.docker.com',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+    'X-DOCKER-API-CLIENT': 'docker-hub/v3012.0.0',
+}
+base = pathlib.Path(__file__).parent.parent.parent.absolute()
+template = '''# Vulhub image for {placeholder_name}
+
+## Image Description
+
+Vulhub is an open-source collection of pre-built vulnerable docker environments. No pre-existing knowledge of docker is required, just execute two simple commands and you have a vulnerable environment.
+
+This image is one of the environment of Vulhub project.
+
+Please see the [USER MANUAL](https://github.com/vulhub/vulhub) from the Vulhub project to see more detail.
+
+## Usage
+
+Do not use this image directly, use it through [docker compose](https://docs.docker.com/compose/).
+
+```
+docker compose up -d
+```
+
+Following environments are using this image, you can find the `docker-compose.yml` file on these folders:
+
+{placeholder_vulns_block}
+
+## Quick reference
+
+- **Maintained by** <br>[phith0n](https://github.com/phith0n) and other contributors from [Vulhub](https://github.com/vulhub)
+- **Where to get help:** <br>[Github Issues](https://github.com/vulhub/vulhub/issues)
+- **Which environments do this image be used:**<br>{placeholder_vulns_inline}
+
+## License
+
+Because Vulhub is packaged with other software, please refer to the software license for the software inside the Vulhub image.
+
+Vulhub's own code is open source based on the [MIT license](https://github.com/vulhub/vulhub/blob/master/LICENSE).
+
+As for any pre-built image usage, it is the image user's responsibility to ensure that any use of this image complies with any relevant licenses for all software contained within.
+'''
+
+
+def prepare_vulhub() -> Mapping:
+    vulhub = defaultdict(list)
+    for f in base.rglob("docker-compose.yml"):
+        start = len(str(base.absolute())) + 1
+        end = len(str(f.absolute())) - 18 - 1
+        vuln_path = str(f.absolute())[start:end].replace('\\', '/')
+
+        compose = yaml.safe_load(f.read_bytes())
+        for service_name, array in compose['services'].items():
+            if 'image' not in array:
+                continue
+
+            image_name = array['image']
+            if ':' in image_name:
+                image_name, _ = image_name.split(':')
+
+            if image_name.startswith('vulhub/'):
+                vulhub[image_name].append(vuln_path)
+
+    return vulhub
+
+
+def build_readme(name, vulns: Iterable):
+    envs = []
+    for vuln in vulns:
+        envs.append(f'[{vuln}](https://github.com/vulhub/vulhub/tree/master/{vuln})')
+
+    return template\
+            .replace('{placeholder_name}', name)\
+            .replace('{placeholder_vulns_block}', '\n'.join([f'- {v}' for v in envs]))\
+            .replace('{placeholder_vulns_inline}', ', '.join(envs))
+
+
+def list_all_repository():
+    response = session.get('https://hub.docker.com/v2/repositories/vulhub?page_size=200&ordering=last_updated')
+    data = response.json()
+    if response.status_code != 200 or data.get('error', None):
+        raise Exception('authentication error')
+
+    for obj in data['results']:
+        yield f"{obj['namespace']}/{obj['name']}"
+
+
+def update_description(name, vulns):
+    response = session.patch(f'https://hub.docker.com/v2/repositories/{name}/', json={
+        'full_description': build_readme(name, vulns)
+    }, headers={
+        'Content-Type': 'application/json'
+    })
+    if response.status_code != 200:
+        raise Exception(f'update readme for {name} failed, status code = {response.status_code}, response text = {response.text}')
+
+
+def main():
+    try:
+        vuln = prepare_vulhub()
+        for name in list_all_repository():
+            update_description(name, vuln.get(name, []))
+            logging.info("Success to update readme for %s", name)
+    except Exception as e:
+        logging.error("error: %r", e, exc_info=True)
+
+
+if __name__ == '__main__':
+    main()