Merge branch 'yaml' into 'master'

YAML (Central Knowledge Base) + Pre-Release + GitLab Pages

See merge request kalilinux/build-scripts/kali-arm!333

.gitignore

@@ -6,3 +6,9 @@ images/
 # Files
 builder.txt
 .DS_Store
+# GitLab pages
+public/
+# Generated documentation
+kernel*.md
+image*.md
+device*.sh

.gitlab-ci.yml

@@ -0,0 +1,52 @@
+# REF: https://gitlab.com/kalilinux/nethunter/build-scripts/kali-nethunter-devices/-/blob/cf3ca7ae/.gitlab-ci.yml
+
+## Docker image: https://hub.docker.com/_/python
+image: docker.io/python:3.7
+
+stages:
+  - linting
+  - generate_documentation
+
+.install_prerequesites_pip: &install_prerequesites_pip |
+  pip install -r .gitlab/requirements.txt
+
+.setup_for_html: &setup_for_html |
+  apt-get update
+  apt-get -y install pandoc
+
+linting:
+  stage: linting
+  allow_failure: yes
+  rules:
+    - if: $CI_MERGE_REQUEST_ID             # Execute jobs in merge request context
+    - if: $CI_COMMIT_BRANCH == 'master'    # Execute jobs when a new commit is pushed to master branch
+  script:
+    - *install_prerequesites_pip
+    - yamllint devices.yml
+
+pages:
+  stage: generate_documentation
+  rules:
+    - if: $CI_COMMIT_BRANCH == 'master'    # Execute jobs when a new commit is pushed to master branch
+  script:
+    - *install_prerequesites_pip
+    - *setup_for_html
+    - ./bin/generate_devices_stats.py
+    - ./bin/generate_devices_table.py
+    - ./bin/generate_images_overview.py
+    - ./bin/generate_images_stats.py
+    - ./bin/generate_images_table.py
+    - ./bin/generate_kernel_stats.py
+    - mkdir -p ./public/
+    - cp ./.gitlab/public.css ./public/
+    - pandoc -s ./.gitlab/www.md    -o ./public/index.html          -c public.css
+    - pandoc -s ./device-stats.md   -o ./public/device-stats.html   -c public.css
+    - pandoc -s ./devices.md        -o ./public/devices.html        -c public.css
+    - pandoc -s ./image-overview.md -o ./public/image-overview.html -c public.css
+    - pandoc -s ./image-stats.md    -o ./public/image-stats.html    -c public.css
+    - pandoc -s ./images.md         -o ./public/images.html         -c public.css
+    - pandoc -s ./kernel-stats.md   -o ./public/kernel-stats.html   -c public.css
+  artifacts:
+    paths:
+      - ./public
+    expire_in: 1 week

.gitlab/public.css

@@ -0,0 +1,324 @@
+html {
+  font-size: 100%;
+  overflow-y: scroll;
+  -webkit-text-size-adjust: 100%;
+  -ms-text-size-adjust: 100%;
+}
+
+body {
+  color: #444;
+  font-family: Georgia, Palatino, 'Palatino Linotype', Times, 'Times New Roman', serif;
+  font-size: 12px;
+  line-height: 1.7;
+  padding: 1em;
+  margin-left: auto;
+  margin-right: auto;
+  background: #fefefe;
+}
+
+a {
+  color: #0645ad;
+  text-decoration: none;
+}
+
+a:visited {
+  color: #0b0080;
+}
+
+a:hover {
+  color: #06e;
+}
+
+a:active {
+  color: #faa700;
+}
+
+a:focus {
+  outline: thin dotted;
+}
+
+*::-moz-selection {
+  background: rgba(255, 255, 0, 0.3);
+  color: #000;
+}
+
+*::selection {
+  background: rgba(255, 255, 0, 0.3);
+  color: #000;
+}
+
+a::-moz-selection {
+  background: rgba(255, 255, 0, 0.3);
+  color: #0645ad;
+}
+
+a::selection {
+  background: rgba(255, 255, 0, 0.3);
+  color: #0645ad;
+}
+
+p {
+  margin: 1em 0;
+}
+
+img {
+  max-width: 100%;
+}
+
+h1, h2, h3, h4, h5, h6 {
+  color: #111;
+  line-height: 125%;
+  margin-top: 2em;
+  font-weight: normal;
+}
+
+h4, h5, h6 {
+  font-weight: bold;
+}
+
+h1 {
+  font-size: 2.5em;
+}
+
+h2 {
+  font-size: 2em;
+}
+
+h3 {
+  font-size: 1.5em;
+}
+
+h4 {
+  font-size: 1.2em;
+}
+
+h5 {
+  font-size: 1em;
+}
+
+h6 {
+  font-size: 0.9em;
+}
+
+blockquote {
+  color: #666666;
+  margin: 0;
+  padding-left: 3em;
+  border-left: 0.5em #EEE solid;
+}
+
+hr {
+  display: block;
+  height: 2px;
+  border: 0;
+  border-top: 1px solid #aaa;
+  border-bottom: 1px solid #eee;
+  margin: 1em 0;
+  padding: 0;
+}
+
+pre, code, kbd, samp {
+  color: #000;
+  font-family: monospace, monospace;
+  _font-family: 'courier new', monospace;
+  font-size: 0.98em;
+}
+
+pre {
+  white-space: pre;
+  white-space: pre-wrap;
+  word-wrap: break-word;
+}
+
+b, strong {
+  font-weight: bold;
+}
+
+dfn {
+  font-style: italic;
+}
+
+ins {
+  background: #ff9;
+  color: #000;
+  text-decoration: none;
+}
+
+mark {
+  background: #ff0;
+  color: #000;
+  font-style: italic;
+  font-weight: bold;
+}
+
+sub, sup {
+  font-size: 75%;
+  line-height: 0;
+  position: relative;
+  vertical-align: baseline;
+}
+
+sup {
+  top: -0.5em;
+}
+
+sub {
+  bottom: -0.25em;
+}
+
+ul, ol {
+  margin: 1em 0;
+  padding: 0 0 0 2em;
+}
+
+li p:last-child {
+  margin-bottom: 0;
+}
+
+ul ul, ol ol {
+  margin: .3em 0;
+}
+
+dl {
+  margin-bottom: 1em;
+}
+
+dt {
+  font-weight: bold;
+  margin-bottom: .8em;
+}
+
+dd {
+  margin: 0 0 .8em 2em;
+}
+
+dd:last-child {
+  margin-bottom: 0;
+}
+
+img {
+  border: 0;
+  -ms-interpolation-mode: bicubic;
+  vertical-align: middle;
+}
+
+figure {
+  display: block;
+  text-align: center;
+  margin: 1em 0;
+}
+
+figure img {
+  border: none;
+  margin: 0 auto;
+}
+
+figcaption {
+  font-size: 0.8em;
+  font-style: italic;
+  margin: 0 0 .8em;
+}
+
+table {
+  margin-bottom: 2em;
+  border-bottom: 1px solid #ddd;
+  border-right: 1px solid #ddd;
+  border-spacing: 0;
+  border-collapse: collapse;
+}
+
+table th {
+  padding: .2em 1em;
+  background-color: #eee;
+  border-top: 1px solid #ddd;
+  border-left: 1px solid #ddd;
+}
+
+table td {
+  padding: .2em 1em;
+  border-top: 1px solid #ddd;
+  border-left: 1px solid #ddd;
+  vertical-align: top;
+}
+
+.author {
+  font-size: 1.2em;
+  text-align: center;
+}
+
+@media only screen and (min-width: 480px) {
+  body {
+    font-size: 14px;
+  }
+}
+@media only screen and (min-width: 768px) {
+  body {
+    font-size: 16px;
+  }
+}
+@media print {
+  * {
+    background: transparent !important;
+    color: black !important;
+    filter: none !important;
+    -ms-filter: none !important;
+  }
+
+  body {
+    font-size: 12pt;
+    max-width: 100%;
+  }
+
+  a, a:visited {
+    text-decoration: underline;
+  }
+
+  hr {
+    height: 1px;
+    border: 0;
+    border-bottom: 1px solid black;
+  }
+
+  a[href]:after {
+    content: " (" attr(href) ")";
+  }
+
+  abbr[title]:after {
+    content: " (" attr(title) ")";
+  }
+
+  .ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after {
+    content: "";
+  }
+
+  pre, blockquote {
+    border: 1px solid #999;
+    padding-right: 1em;
+    page-break-inside: avoid;
+  }
+
+  tr, img {
+    page-break-inside: avoid;
+  }
+
+  img {
+    max-width: 100% !important;
+  }
+
+  @page :left {
+    margin: 15mm 20mm 15mm 10mm;
+}
+
+  @page :right {
+    margin: 15mm 10mm 15mm 20mm;
+}
+
+  p, h2, h3 {
+    orphans: 3;
+    widows: 3;
+  }
+
+  h2, h3 {
+    page-break-after: avoid;
+  }
+}

.gitlab/requirements.txt

@@ -0,0 +1 @@
+yamllint

.gitlab/www.md

@@ -0,0 +1,24 @@
+---
+title: Kali ARM Statistics
+---
+
+## Devices
+
+- [Devices](devices.html)
+- [Device Stats](device-stats.html)
+
+## Images
+
+- [Images](images.html)
+- [Image Overview](image-overview.html)
+- [Image Stats](image-stats.html)
+
+## Kernels
+
+- [Kernel Stats](kernel-stats.html)
+
+- - -
+
+## Links
+
+- [Kali Home](https://www.kali.org/)

.yamllint.yml

@@ -0,0 +1,7 @@
+extends: default
+
+rules:
+  # 250 chars should be enough, but don't fail if a line is longer
+  line-length:
+    max: 250
+    level: warning

bin/generate_devices_stats.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+import yaml # python3 -m pip install pyyaml --user
+from datetime import datetime
+import sys
+
+OUTPUT_FILE = './device-stats.md'
+INPUT_FILE = './devices.yml'
+repo_msg = "\n_This table was generated automatically on {} from the [Kali ARM GitLab repository](https://gitlab.com/kalilinux/build-scripts/kali-arm)_\n".format(datetime.now().strftime("%Y-%B-%d %H:%M:%S"))
+qty_devices = 0
+qty_images = 0
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def generate_table(data):
+    global qty_devices, qty_images
+    default = ""
+    table  = "| Vendor | Board | Images |\n"
+    table += "|--------|-------|--------|\n"
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                qty_devices += 1
+                qty_images += len(board.get('images', default))
+                table += "| {} | {} | {} |\n".format(vendor,
+                                                     board.get('board', default),
+                                                     len(board.get('images', default)))
+    return table
+
+def read_file(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except Exception as e:
+        print("[-] Cannot open input file: {} - {}".format(file, e))
+    return data
+
+def write_file(data, file):
+    try:
+        with open(file, 'w') as f:
+            meta  = '---\n'
+            meta += 'title: Kali ARM Device Statistics\n'
+            meta += '---\n\n'
+            stats  = "- The official [Kali ARM repository](https://gitlab.com/kalilinux/build-scripts/kali-arm) contains build-scripts to support [**{}** Kali ARM devices](devices.html)\n".format(str(str(qty_devices)))
+            stats += "- [Kali ARM Statistics](index.html)\n\n"
+            f.write(str(meta))
+            f.write(str(stats))
+            f.write(str(data))
+            f.write(str(repo_msg))
+            f.close()
+            print('[+] File: {} successfully written'.format(OUTPUT_FILE))
+    except Exception as e:
+        print("[-] Cannot write to output file: {} - {}".format(file, e))
+    return 0
+
+def print_summary():
+    print('Devices: {}'.format(qty_devices))
+    print('Images : {}'.format(qty_images))
+
+def main(argv):
+    # Assign variables
+    data = read_file(INPUT_FILE)
+
+    # Get data
+    res = yaml_parse(data)
+    generated_markdown = generate_table(res)
+
+    # Create markdown file
+    write_file(generated_markdown, OUTPUT_FILE)
+
+    # Print result
+    print_summary()
+
+    # Exit
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

bin/generate_devices_table.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+import yaml # python3 -m pip install pyyaml --user
+from datetime import datetime
+import sys
+
+OUTPUT_FILE = './devices.md'
+INPUT_FILE = './devices.yml'
+repo_msg = "\n_This table was generated automatically on {} from the [Kali ARM GitLab repository](https://gitlab.com/kalilinux/build-scripts/kali-arm)_\n".format(datetime.now().strftime("%Y-%B-%d %H:%M:%S"))
+qty_devices = 0
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def generate_table(data):
+    global qty_devices
+    default = ""
+    table  = "| Vendor | Board | CPU | CPU Cores | GPU | RAM | RAM Size | Ethernet | Ethernet Speed | Wi-Fi | Bluetooth | USB2 | USB3 | Storage | Notes |\n"
+    table += "|--------|-------|-----|-----------|-----|-----|----------|----------|----------------|-------|-----------|------|------|---------|-------|\n"
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                qty_devices += 1
+                table += "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |\n".format(vendor,
+                                                                                                            board.get('board', default),
+                                                                                                            board.get('cpu', default),
+                                                                                                            board.get('cpu-cores', default),
+                                                                                                            board.get('gpu', default),
+                                                                                                            board.get('ram', default),
+                                                                                                            board.get('ethernet', default),
+                                                                                                            board.get('ethernet-speed', default),
+                                                                                                            board.get('wifi', default),
+                                                                                                            board.get('bluetooth', default),
+                                                                                                            board.get('usb2', default),
+                                                                                                            board.get('usb3', default),
+                                                                                                            board.get('storage', default),
+                                                                                                            board.get('notes', default))
+    return table
+
+def read_file(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except Exception as e:
+        print("[-] Cannot open input file: {} - {}".format(file, e))
+    return data
+
+def write_file(data, file):
+    try:
+        with open(file, 'w') as f:
+            meta  = '---\n'
+            meta += 'title: Kali ARM Devices\n'
+            meta += '---\n\n'
+            stats  = "- The official [Kali ARM repository](https://gitlab.com/kalilinux/build-scripts/kali-arm) contains build-scripts to support [**{}** Kali ARM devices](device-stats.html)\n".format(str(str(qty_devices)))
+            stats += "- [Kali ARM Statistics](index.html)\n\n"
+            f.write(str(meta))
+            f.write(str(stats))
+            f.write(str(data))
+            f.write(str(repo_msg))
+            f.close()
+            print('[+] File: {} successfully written'.format(OUTPUT_FILE))
+    except Exception as e:
+        print("[-] Cannot write to output file: {} - {}".format(file, e))
+    return 0
+
+def print_summary():
+    print('Devices        : {}'.format(qty_devices))
+
+def main(argv):
+    # Assign variables
+    data = read_file(INPUT_FILE)
+
+    # Get data
+    res = yaml_parse(data)
+    generated_markdown = generate_table(res)
+
+    # Create markdown file
+    write_file(generated_markdown, OUTPUT_FILE)
+
+    # Print result
+    print_summary()
+
+    # Exit
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

bin/generate_images_overview.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# REF: https://www.kali.org/docs/arm/
+import yaml # python3 -m pip install pyyaml --user
+from datetime import datetime
+import sys
+
+OUTPUT_FILE = './image-overview.md'
+INPUT_FILE = './devices.yml'
+repo_msg = "\n_This table was generated automatically on {} from the [Kali ARM GitLab repository](https://gitlab.com/kalilinux/build-scripts/kali-arm)_\n".format(datetime.now().strftime("%Y-%B-%d %H:%M:%S"))
+qty_devices = 0
+qty_images = 0
+qty_image_kali = 0
+qty_image_community = 0
+qty_image_eol = 0
+qty_image_unknown = 0
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def generate_table(data):
+    global qty_devices, qty_images, qty_image_kali, qty_image_community, qty_image_eol, qty_image_unknown
+    images = []
+    default = ""
+    table  = "| [Device](https://www.kali.org/docs/arm/) | [Build-Script](https://gitlab.com/kalilinux/build-scripts/kali-arm/) | [Official Image](https://www.kali.org/get-kali/#kali-arm) | Community Image | Retired Image |\n"
+    table += "|--------|--------------|----------------|-----------------|---------------|\n"
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                qty_devices += 1
+                # Iterate over per board
+                for key in board.keys():
+                    # Check if there is an image for the board
+                    if 'images' in key:
+                        # Iterate over image (depth 3)
+                        for image in board[key]:
+                            if image['name'] not in images:
+                                images.append(image['name']) # ALT: images.append(image['image'])
+                                qty_images += 1
+                                build_script = image.get('build-script', default)
+                                if build_script:
+                                    build_script = "[{0}](https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/{0})".format(build_script)
+                                name = image.get('name', default)
+                                slug = image.get('slug', default)
+                                if name and slug:
+                                    name = "[{}](https://www.kali.org/docs/arm/{}/)".format(name, slug)
+                                support = image.get('support', default)
+                                if support == "kali":
+                                    status = "x |  | "
+                                    qty_image_kali += 1
+                                elif support == "community":
+                                    status = " | x | "
+                                    qty_image_community += 1
+                                elif support == "eol":
+                                    status = " |  | x"
+                                    qty_image_eol += 1
+                                else:
+                                    status = " |  | "
+                                    qty_image_unknown += 1
+                                table += "| {} | {} | {} |\n".format(name,
+                                                                     build_script,
+                                                                     status)
+                            #else:
+                            #    print('DUP {} / {}'.format(image['name'], image['image']))
+                if 'images' not in board.keys():
+                    print("[i] Possible issue with: " + board.get('board', default) + " (no images)")
+    return table
+
+def read_file(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except Exception as e:
+        print("[-] Cannot open input file: {} - {}".format(file, e))
+    return data
+
+def write_file(data, file):
+    try:
+        with open(file, 'w') as f:
+            meta  = '---\n'
+            meta += 'title: Kali ARM Image Overview\n'
+            meta += '---\n\n'
+            stats  = "- The official [Kali ARM repository](https://gitlab.com/kalilinux/build-scripts/kali-arm) contains build-scripts to create [**{}** unique Kali ARM images](image-stats.html) for **{}** devices\n".format(str(qty_images), str(qty_devices))
+            stats += "- The [next release](https://www.kali.org/releases/) cycle will include [**{}** Kali ARM images](image-stats.html) _([ready to download](https://www.kali.org/get-kali/#kali-arm))_, **{}** images which can be built, and {} retired images\n".format(str(qty_image_kali), str(qty_image_community), str(qty_image_eol))
+            stats += "- [Kali ARM Statistics](index.html)\n\n"
+            f.write(str(meta))
+            f.write(str(stats))
+            f.write(str(data))
+            f.write(str(repo_msg))
+            f.close()
+            print('[+] File: {} successfully written'.format(OUTPUT_FILE))
+    except Exception as e:
+        print("[-] Cannot write to output file: {} - {}".format(file, e))
+    return 0
+
+def print_summary():
+    print('Devices: {}'.format(qty_devices))
+    print('Images : {}'.format(qty_images))
+    print('- Kali     : {}'.format(qty_image_kali))
+    print('- Community: {}'.format(qty_image_community))
+    print('- EOL      : {}'.format(qty_image_eol))
+    print('- Unknown  : {}'.format(qty_image_unknown))
+def main(argv):
+    # Assign variables
+    data = read_file(INPUT_FILE)
+
+    # Get data
+    res = yaml_parse(data)
+    generated_markdown = generate_table(res)
+
+    # Create markdown file
+    write_file(generated_markdown, OUTPUT_FILE)
+
+    # Print result
+    print_summary()
+
+    # Exit
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

bin/generate_images_stats.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# REF: https://gitlab.com/kalilinux/nethunter/build-scripts/kali-nethunter-devices/-/blob/52cbfb36/scripts/generate_images_stats.py
+import yaml # python3 -m pip install pyyaml --user
+from datetime import datetime
+import sys
+
+OUTPUT_FILE = './image-stats.md'
+INPUT_FILE = './devices.yml'
+repo_msg = "\n_This table was generated automatically on {} from the [Kali ARM GitLab repository](https://gitlab.com/kalilinux/build-scripts/kali-arm)_\n".format(datetime.now().strftime("%Y-%B-%d %H:%M:%S"))
+qty_images = 0
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def generate_table(data):
+    global qty_images
+    images = []
+    default = ""
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                # Iterate over per board
+                for key in board.keys():
+                    # Check if there is an image for the board
+                    if 'images' in key:
+                        # Iterate over image (depth 3)
+                        for image in board[key]:
+                            images.append("{} ({})".format(image.get('name', default),
+                                                           image.get('architecture', default)))
+                if 'images' not in board.keys():
+                    print("[i] Possible issue with: " + board.get('board', default) + " (no images)")
+
+    table  = "| Image Name (Architecture) |\n"
+    table += "|---------------------------|\n"
+    # iterate over all the devices
+    for device in sorted(set(images)):
+        table += "| {} |\n".format(device)
+    qty_images = len(set(images))
+    return table
+
+def read_file(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except Exception as e:
+        print("[-] Cannot open input file: {} - {}".format(file, e))
+    return data
+
+def write_file(data, file):
+    try:
+        with open(file, 'w') as f:
+            meta  = '---\n'
+            meta += 'title: Kali ARM Image Statistics\n'
+            meta += '---\n\n'
+            stats  = "- The official [Kali ARM repository](https://gitlab.com/kalilinux/build-scripts/kali-arm) contains build-scripts to create [**{}** unique Kali ARM images](images.html)\n".format(str(qty_images))
+            stats += "- [Kali ARM Statistics](index.html)\n\n"
+            f.write(str(meta))
+            f.write(str(stats))
+            f.write(str(data))
+            f.write(str(repo_msg))
+            f.close()
+            print('[+] File: {} successfully written'.format(OUTPUT_FILE))
+    except Exception as e:
+        print("[-] Cannot write to output file: {} - {}".format(file, e))
+    return 0
+
+def print_summary():
+    print('Images: {}'.format(qty_images))
+
+def main(argv):
+    # Assign variables
+    data = read_file(INPUT_FILE)
+
+    # Get data
+    res = yaml_parse(data)
+    generated_markdown = generate_table(res)
+
+    # Create markdown file
+    write_file(generated_markdown, OUTPUT_FILE)
+
+    # Print result
+    print_summary()
+
+    # Exit
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

bin/generate_images_table.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# REF: https://gitlab.com/kalilinux/nethunter/build-scripts/kali-nethunter-devices/-/blob/95ad7d2b/scripts/generate_images_table.py
+import yaml # python3 -m pip install pyyaml --user
+from datetime import datetime
+import sys
+
+OUTPUT_FILE = './images.md'
+INPUT_FILE = './devices.yml'
+repo_msg = "\n_This table was generated automatically on {} from the [Kali ARM GitLab repository](https://gitlab.com/kalilinux/build-scripts/kali-arm)_\n".format(datetime.now().strftime("%Y-%B-%d %H:%M:%S"))
+qty_devices = 0
+qty_images = 0
+qty_images_released = 0
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def generate_table(data):
+    global qty_devices, qty_images, qty_images_released
+    images = []
+    images_released = []
+    default = ""
+    table  = "| Image Name | Filename | Architecture | Preferred | Support | Documentation | Kernel | Kernel Version | Notes |\n"
+    table += "|------------|----------|--------------|-----------|---------|---------------|--------|----------------|-------|\n"
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                qty_devices += 1
+                # Iterate over per board
+                for key in board.keys():
+                    # Check if there is an image for the board
+                    if 'images' in key:
+                        # Iterate over image (depth 3)
+                        for image in board[key]:
+                            #qty_images += 1
+                            images.append("{}".format(image.get('name', default)))
+                            support = image.get('support', default)
+                            if support == "kali":
+                                #qty_images_released += 1
+                                images_released.append("{}".format(image.get('name', default)))
+                            slug = image.get('slug', default)
+                            if slug:
+                                slug = "[{0}](https://www.kali.org/docs/arm/{0})".format(slug)
+                            table += "| {} | {} | {} | {} | {} | {} | {} | {} |\n".format(image.get('name', default),
+                                                                                          image.get('image', default),
+                                                                                          image.get('architecture', default),
+                                                                                          image.get('recommended', default),
+                                                                                          image.get('support', default),
+                                                                                          slug,
+                                                                                          image.get('kernel', default),
+                                                                                          image.get('kernel-version', default),
+                                                                                          image.get('image-notes', default))
+                if 'images' not in board.keys():
+                    print("[i] Possible issue with: " + board.get('board', default) + " (no images)")
+    qty_images = len(set(images))
+    qty_images_released = len(set(images_released))
+    return table
+
+def read_file(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except Exception as e:
+        print("[-] Cannot open input file: {} - {}".format(file, e))
+    return data
+
+def write_file(data, file):
+    try:
+        with open(file, 'w') as f:
+            meta  = '---\n'
+            meta += 'title: Kali ARM Images\n'
+            meta += '---\n\n'
+            stats  = "- The official [Kali ARM repository](https://gitlab.com/kalilinux/build-scripts/kali-arm) contains build-scripts to create [**{}** unique Kali ARM images](image-stats.html) for **{}** devices\n".format(str(qty_images), str(qty_devices))
+            stats += "- The [next release](https://www.kali.org/releases/) cycle will include [**{}** Kali ARM images](image-stats.html) _([ready to download](https://www.kali.org/get-kali/#kali-arm))_\n".format(str(qty_images_released))
+            stats += "- [Kali ARM Statistics](index.html)\n\n"
+            f.write(str(meta))
+            f.write(str(stats))
+            f.write(str(data))
+            f.write(str(repo_msg))
+            f.close()
+            print('[+] File: {} successfully written'.format(OUTPUT_FILE))
+    except Exception as e:
+        print("[-] Cannot write to output file: {} - {}".format(file, e))
+    return 0
+
+def print_summary():
+    print('Devices        : {}'.format(qty_devices))
+    print('Images         : {}'.format(qty_images))
+    print('Images Released: {}'.format(qty_images_released))
+
+def main(argv):
+    # Assign variables
+    data = read_file(INPUT_FILE)
+
+    # Get data
+    res = yaml_parse(data)
+    generated_markdown = generate_table(res)
+
+    # Create markdown file
+    write_file(generated_markdown, OUTPUT_FILE)
+
+    # Print result
+    print_summary()
+
+    # Exit
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

bin/generate_kernel_stats.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# REF: https://gitlab.com/kalilinux/nethunter/build-scripts/kali-nethunter-devices/-/blob/52cbfb36/scripts/generate_images_stats.py
+import yaml # python3 -m pip install pyyaml --user
+from datetime import datetime
+import sys
+
+OUTPUT_FILE = './kernel-stats.md'
+INPUT_FILE = './devices.yml'
+repo_msg = "\n_This table was generated automatically on {} from the [Kali ARM GitLab repository](https://gitlab.com/kalilinux/build-scripts/kali-arm)_\n".format(datetime.now().strftime("%Y-%B-%d %H:%M:%S"))
+qty_kernels = 0
+qty_versions = {
+                'custom':  0,
+                'kali':    0,
+                'vendor':  0,
+                'unknown': 0
+               }
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def generate_table(data):
+    global qty_kernels, qty_versions
+    images = []
+    default = "unknown"
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                # Iterate over per board
+                for key in board.keys():
+                    # Check if there is an image for the board
+                    if 'images' in key:
+                        # Iterate over image (depth 3)
+                        for image in board[key]:
+                            if image['name'] not in images:
+                                images.append(image['name']) # ALT: images.append(image['image'])
+                                qty_kernels += 1
+                                qty_versions[(image.get('kernel', default))] += 1
+                            #else:
+                            #    print('DUP {} / {}'.format(image['name'], image['image']))
+                if 'images' not in board.keys():
+                    print("[i] Possible issue with: " + board.get('board', default) + " (no images)")
+
+    table  = "| Kernel | Qty |\n"
+    table += "|--------|-----|\n"
+
+    # iterate over all the devices
+    for v in qty_versions:
+        table += "| {} | {} |\n".format(v.capitalize(),
+                                        str(qty_versions[v]))
+    return table
+
+def read_file(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except Exception as e:
+        print("[-] Cannot open input file: {} - {}".format(file, e))
+    return data
+
+def write_file(data, file):
+    try:
+        with open(file, 'w') as f:
+            meta  = '---\n'
+            meta += 'title: Kali ARM Kernel Statistics\n'
+            meta += '---\n\n'
+            stats  = "- The official [Kali ARM repository](https://gitlab.com/kalilinux/build-scripts/kali-arm) contains build-scripts to create [**{}** unique Kali ARM images](images.html)\n".format(str(qty_kernels))
+            stats += "- [Kali ARM Statistics](index.html)\n\n"
+            f.write(str(meta))
+            f.write(str(stats))
+            f.write(str(data))
+            f.write(str(repo_msg))
+            f.close()
+            print('[+] File: {} successfully written'.format(OUTPUT_FILE))
+    except Exception as e:
+        print("[-] Cannot write to output file: {} - {}".format(file, e))
+    return 0
+
+def print_summary():
+    print('Kernels: {}'.format(qty_kernels))
+
+def main(argv):
+    # Assign variables
+    data = read_file(INPUT_FILE)
+
+    # Get data
+    res = yaml_parse(data)
+    generated_markdown = generate_table(res)
+
+    # Create markdown file
+    write_file(generated_markdown, OUTPUT_FILE)
+
+    # Print result
+    print_summary()
+
+    # Exit
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

bin/pre-release.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+## NetHunter ~ https://gitlab.com/kalilinux/nethunter/build-scripts/kali-nethunter-project/-/blob/2e26ee29/nethunter-installer/prep-release.py
+## ARM Devices ~ https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+###############################################
+## Script to prepare Kali ARM quarterly release
+##
+## This should be run after images are created.
+##
+## It parses the YAML sections of the devices.yml and creates:
+##
+## - "<outputdir>/manifest.json": manifest file mapping image name to display name
+##
+## Dependencies:
+##   sudo apt -y install python3 python3-yaml
+##
+## Usage:
+##  ./bin/pre-release.py -i <input file> -o <output directory> -r <release>
+##
+## E.g.:
+## ./bin/pre-release.py -i devices.yml -o images/ -r 2022.3
+
+import json
+import datetime
+import yaml # python3 -m pip install pyyaml --user
+import getopt, os, stat, sys
+
+manifest = ""     # Generated automatically (<outputdir>/manifest.json)
+release = ""
+outputdir = ""
+inputfile = ""
+qty_images = 0
+qty_devices = 0
+
+## Input:
+## ------------------------------------------------------------ ##
+## See: ./devices.yml
+##      https://gitlab.com/kalilinux/build-scripts/kali-arm/-/blob/master/devices.yml
+
+def bail(message = "", strerror = ""):
+    outstr = ""
+    prog = sys.argv[0]
+    if message != "":
+        outstr = "\nError: {}".format(message)
+    if strerror != "":
+        outstr += "\nMessage: {}\n".format(strerror)
+    else:
+        outstr += "\n\nUsage: {} -i <input file> -o <output directory> -r <release>".format(prog)
+        outstr += "\nE.g. : {} -i devices.yml -o images/ -r {}.1\n".format(prog, datetime.datetime.now().year)
+    print(outstr)
+    sys.exit(2)
+
+def getargs(argv):
+    global inputfile, outputdir, release
+
+    try:
+        opts, args = getopt.getopt(argv,"hi:o:r:",["inputfile=","outputdir=","release="])
+    except getopt.GetoptError as e:
+        bail("Incorrect arguments: {}".format(e))
+
+    if opts:
+        for opt, arg in opts:
+            if opt == '-h':
+                bail()
+            elif opt in ("-i", "--inputfile"):
+                inputfile = arg
+            elif opt in ("-o", "--outputdir"):
+                outputdir = arg.rstrip("/")
+            elif opt in ("-r", "--release"):
+                release = arg
+            else:
+                bail("Unrecognised argument: " + opt)
+    else:
+        bail("Failed to read arguments")
+
+    if not release:
+        bail("Missing required argument: -r/--release")
+    return 0
+
+def yaml_parse(content):
+    result = ""
+    lines = content.split('\n')
+    for line in lines:
+        if line.strip() and not line.strip().startswith('#'):
+            result += line + "\n"
+    return yaml.safe_load(result)
+
+def jsonarray(devices, vendor, name, filename, preferred, slug):
+    if not vendor in devices:
+        devices[vendor] = []
+    jsondata = {"name": name, "filename": filename, "preferred": preferred, "slug": slug}
+    devices[vendor].append(jsondata)
+    return devices
+
+def generate_manifest(data):
+    global release, qty_devices, qty_images
+    default = ""
+    devices = {}
+
+    # Iterate over per input (depth 1)
+    for yaml in data['devices']:
+        # Iterate over vendors
+        for vendor in yaml.keys():
+            # Iterate over board (depth 2)
+            for board in yaml[vendor]:
+                qty_devices += 1
+                # Iterate over per board
+                for key in board.keys():
+                    # Check if there is an image for the board
+                    if 'images' in key:
+                        # Iterate over image (depth 3)
+                        for image in board[key]:
+                            qty_images += 1
+                            name = image.get('name', default)
+                            filename = "kali-linux-{}-{}".format(release, image.get('image', default))
+                            preferred = image.get('preferred-image', default)
+                            slug = image.get('slug', default)
+                            jsonarray(devices, vendor, name, filename, preferred, slug)
+    return json.dumps(devices, indent = 2)
+
+def deduplicate(data):
+    # Remove duplicate lines
+    clean_data = ""
+    lines_seen = set()
+    for line in data.splitlines():
+        if line not in lines_seen: # not a duplicate
+            clean_data += line + "\n"
+            lines_seen.add(line)
+    return clean_data
+
+def createdir(dir):
+    try:
+        if not os.path.exists(dir):
+            os.makedirs(dir)
+    except:
+        bail('Directory "' + dir + '" does not exist and cannot be created')
+    return 0
+
+def readfile(file):
+    try:
+        with open(file) as f:
+            data = f.read()
+            f.close()
+    except:
+        bail("Cannot open input file: " + file)
+    return data
+
+def writefile(data, file):
+    try:
+        with open(file, 'w') as f:
+            f.write(str(data))
+            f.close()
+    except:
+        bail("Cannot write to output file: " + file)
+    return 0
+
+def main(argv):
+    global inputfile, outputdir, release
+
+    # Parse command-line arguments
+    if len(sys.argv) > 1:
+        getargs(argv)
+    else:
+        bail("Missing arguments")
+
+    # Assign variables
+    manifest = outputdir + "/manifest.json"
+    data = readfile(inputfile)
+
+    # Get data
+    res = yaml_parse(data)
+    manifest_list = generate_manifest(res)
+
+    # Create output directory if required
+    createdir(outputdir)
+
+    # Create manifest file
+    writefile(manifest_list, manifest)
+
+    # Print result and exit
+    print('\nStats:')
+    print('  - Devices\t: {}'.format(qty_devices))
+    print('  - Images\t: {}'.format(qty_images))
+    print("\n")
+    print('Manifest file created\t: {}'.format(manifest))
+
+    exit(0)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])