jcm/pkg.py

#!/usr/bin/env python3
# coding=utf-8

"""
JCM Package Builder - JCM 包构建工具

这个模块是 JCM (Jiang Cluster Management) 项目的核心包管理工具，负责：
1. 从 server/ 目录构建 .pkg 包文件
2. 生成包索引文件 (Packages)
3. 管理包依赖关系和版本信息
4. 支持 Windows 可执行文件生成
5. 提供测试包构建功能

主要功能：
- Build_pkg(): 构建单个包
- 包索引生成和 MD5 校验
- 跨平台支持 (Linux/Windows)
- 动态模块加载
- 文件操作和目录管理

作者: Jiang
项目: JCM Server Tools
许可: GPL v2
"""

import os
import sys
import shutil
import tarfile
import hashlib
import json
import csv
from pathlib import Path

class JCMPackageBuilder:
    """
    JCM 包构建器主类
    
    负责管理整个包构建流程，包括包的创建、索引生成和文件管理
    """
    
    def __init__(self):
        """
        初始化包构建器
        
        设置基本的目录路径和清理现有的包索引文件
        """
        self.jcm_dir = Path(".jcm")
        self.out_dir = Path(".out") 
        self.server_dir = Path("server")
        
        # 设置排除目录
        self.excluded_dirs = {
            'APP', 'app',
            'info', 'Info', 
            'setup', 'Setup',
            'tocken', 'token',
            '__pycache__',
            '.git', '.svn',
            'test', 'Test',
            'temp', 'tmp'
        }
        
        # 清理现有的 Packages 文件
        self._clean_existing_packages()
    
    def _clean_existing_packages(self):
        """
        清理现有的包索引文件
        
        遍历 .jcm 目录，删除所有现有的 Packages 文件
        """
        if self.jcm_dir.exists():
            for item in self.jcm_dir.iterdir():
                if item.is_dir():
                    packages_file = item / "Packages"
                    if packages_file.exists():
                        packages_file.unlink()
                        print(f"已清理现有包索引文件: {packages_file}")

    def build_package(self, name):
        """
        构建单个包
        
        Args:
            name (str): 包名称，对应 server/ 目录下的子目录名
            
        Returns:
            bool: 构建是否成功
            
        Raises:
            FileNotFoundError: 当包目录或 Package.py 不存在时
            ValueError: 当包信息缺少必需字段时
        """
        try:
            package_path = self.server_dir / name
            
            # 检查包目录是否存在
            if not package_path.is_dir():
                print(f"警告: {package_path} 不是一个有效的目录")
                return False
            
            # 检查 Package.py 文件是否存在
            package_py = package_path / "Package.py"
            if not package_py.exists():
                print(f"警告: 未找到包配置文件 {package_py}")
                return False
            
            # 动态加载包配置模块
            pkg_module = self.load_module('run', str(package_py))
            
            # 获取包信息
            data = pkg_module.info()
            
            # 验证必需字段
            required_fields = [
                "Package", "Version", "names", "name", 
                "Depends", "License", "issued", "Description", "Descriptions"
            ]
            
            for field in required_fields:
                if field not in data:
                    raise ValueError(f"缺少必需字段: {field}")
            
            # 创建包文件
            return self._create_package_archive(data, pkg_module)
            
        except Exception as e:
            print(f"构建包 {name} 时发生错误: {e}")
            return False

    def _create_package_archive(self, data, pkg_module):
        """
        创建包归档文件
        
        Args:
            data (dict): 包信息字典
            pkg_module: 包模块对象
            
        Returns:
            bool: 创建是否成功
        """
        try:
            package_name = data["Package"]
            version = data["Version"]
            issued = data["issued"]
            
            print(f"正在构建: {package_name}_{version}.pkg",end="\t")
            
            # 清理并创建输出目录
            if self.out_dir.exists():
                shutil.rmtree(self.out_dir)
            
            self._safe_mkdir(self.out_dir)
            self._safe_mkdir(self.out_dir / "server")
            self._safe_mkdir(self.out_dir / "web")
            self._safe_mkdir(self.out_dir / "web" / "Ace_Admin")
            
            # 调用包的输出方法
            pkg_module.out()
            
            # 创建 tar.gz 归档
            shutil.make_archive(".pkg", "gztar", base_dir=str(self.out_dir))
            
            # 创建目标目录并移动包文件
            target_dir = self.jcm_dir / issued
            self._safe_mkdir(target_dir)
            
            pkg_filename = f"{package_name}_{version}.pkg"
            source_file = Path(".pkg.tar.gz")
            target_file = target_dir / pkg_filename
            
            shutil.move(str(source_file), str(target_file))
            
            print(f"[ OK ]")
            return True
            
        except Exception as e:
            print(f"创建包归档时发生错误: {e}")
            return False

    def _safe_mkdir(self, path):
        """
        安全创建目录
        
        Args:
            path (Path): 要创建的目录路径
        """
        Path(path).mkdir(parents=True, exist_ok=True)

    def load_module(self, name, file_path):
        """
        动态加载 Python 模块
        
        支持不同 Python 版本的模块加载方式，优先使用新的 importlib.util
        
        Args:
            name (str): 模块名称
            file_path (str): 模块文件路径
            
        Returns:
            module: 加载的模块对象
            
        Raises:
            ImportError: 当模块加载失败时
        """
        try:
            # Python 3.4+ 推荐方式
            import importlib.util
            spec = importlib.util.spec_from_file_location(name, file_path)
            if spec is None:
                raise ImportError(f"无法从 {file_path} 加载模块规范")
            
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            return module
            
        except (ImportError, AttributeError):
            # 回退到旧方法
            try:
                import importlib.machinery
                return importlib.machinery.SourceFileLoader(name, file_path).load_module()
            except ImportError:
                # 最后的回退方案 (Python < 3.4)
                import imp
                return imp.load_source(name, file_path)

    def build_all_packages(self):
        """
        构建所有包
        
        遍历 server/ 目录，为每个子目录构建对应的包
        """
        print("开始包构建流程...")
        
        if not self.server_dir.exists():
            print(f"错误: 未找到 {self.server_dir} 目录")
            return
        
        # 构建所有包
        success_count = 0
        total_count = 0
        
        for package_dir in self.server_dir.iterdir():
            if package_dir.is_dir():
                # 静默跳过排除目录
                if package_dir.name in self.excluded_dirs:
                    continue
                
                total_count += 1
                if self.build_package(package_dir.name):
                    success_count += 1
        
        print(f"包构建完成: {success_count}/{total_count} 个包构建成功")
        
        # 生成包索引
        self.generate_package_indexes()
        
        # 清理临时文件
        self._cleanup_temp_files()
        
        print("包构建流程已完成")

    def generate_package_indexes(self):
        """
        生成包索引文件
        
        为每个包类别生成 Packages 索引文件，包含包的元数据和 MD5 校验和
        """
        print("正在生成包索引文件...")
        
        if not self.jcm_dir.exists():
            print("未找到需要索引的包")
            return
        
        for category_dir in self.jcm_dir.iterdir():
            if not category_dir.is_dir():
                continue
            
            print(f"正在为类别 '{category_dir.name}' 生成索引")
            packages_file = category_dir / "Packages.csv"
            
            # 删除现有的索引文件
            if packages_file.exists():
                packages_file.unlink()
            
            # 生成新的索引文件
            with open(packages_file, mode="w",encoding="utf-8",newline="") as f:
                fields = ["Package", "name", "names", "Version", "Depends", "License", "md5", "Description", "Descriptions"]
                csv_dict_writer = csv.DictWriter(f, fieldnames=fields)
                csv_dict_writer.writeheader()
                pkg_count = 0
                for pkg_file in category_dir.glob("*.pkg"):
                    self._write_package_info(csv_dict_writer, pkg_file, category_dir.name)
                    pkg_count += 1
                
                if pkg_count > 0:
                    print(f"已为类别 '{category_dir.name}' 索引 {pkg_count} 个包")

    def _find_package_py(self, package_name):
        """
        查找 Package.py 文件的正确路径
        
        Args:
            package_name (str): 包名称
        
        Returns:
            Path: Package.py 文件的路径，如果未找到则返回 None
        """
        possible_paths = [
            self.out_dir / self.out_dir / "server" / package_name / "Package.py",  # 重复目录结构
            self.out_dir / "server" / package_name / "Package.py",                # 标准结构
            self.out_dir / ".out" / "server" / package_name / "Package.py",       # .out 前缀
        ]
        
        for path in possible_paths:
            if path.exists():
                return path
        
        return None

    def _write_package_info(self, csv_dict_writer, pkg_file, category):
        """
        写入包信息到索引文件
        
        Args:
            csv_dict_writer: CSV 字典写入器对象
            pkg_file (Path): 包文件路径
            category (str): 包类别
        """
        try:
            # 提取包到临时目录
            if self.out_dir.exists():
                shutil.rmtree(self.out_dir)
            self._safe_mkdir(self.out_dir)
            
            # 解压包文件
            with tarfile.open(pkg_file, 'r:gz') as tar:
                tar.extractall(path=self.out_dir)
            
            # 获取包名（从文件名提取）
            package_name = pkg_file.stem.split('_')[0]
            
            # 查找 Package.py 文件
            package_py = self._find_package_py(package_name)
            
            if package_py is None:
                print(f"警告: 包 {package_name} 的 Package.py 文件未找到")
                return
            
            # 加载包信息
            pkg_module = self.load_module('run', str(package_py))
            data = pkg_module.info()
            
            # 计算 MD5 校验和
            md5_hash = self._calculate_md5(pkg_file)
            
            # 写入包信息
            self._write_package_metadata(csv_dict_writer, data, md5_hash)
            
            print(f"已索引包: {pkg_file.name}")
            
        except Exception as e:
            print(f"索引包 {pkg_file} 时发生错误: {e}")

    def _calculate_md5(self, file_path):
        """
        计算文件的 MD5 校验和
        
        Args:
            file_path (Path): 文件路径
            
        Returns:
            str: MD5 校验和的十六进制字符串
        """
        hash_md5 = hashlib.md5()
        with open(file_path, 'rb') as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hash_md5.update(chunk)
        return hash_md5.hexdigest()

    def _write_package_metadata(self, csv_dict_writer, data, md5_hash):
        """
        写入包元数据到索引文件
        
        Args:
            csv_dict_writer: 文件句柄
            data (dict): 包信息字典
            md5_hash (str): MD5 校验和
        """
        # 写入基本信息
        data = {
            "Package": data["Package"],
            "name": data["name"],
            "names": json.dumps(data["names"]),
            "Version": data["Version"],
            "Depends": data["Depends"],
            "License": data["License"],
            "md5": md5_hash,
            "Description": data["Description"],
            "Descriptions": json.dumps(data["Descriptions"]),
        }
        csv_dict_writer.writerow(data)
        # file_handle.write(f'Package:{data["Package"]}\n'.encode("utf-8"))
        # file_handle.write(f'name:{data["names"]}\n'.encode("utf-8"))
        
        # # 写入多语言名称
        # for key, value in data["namei"].items():
        #     file_handle.write(f'name\t{key}:'.encode("utf-8") + value + b'\n')
        
        # # 写入其他元数据
        # file_handle.write(f'Version:{data["Version"]}\n'.encode("utf-8"))
        # file_handle.write(f'Depends:{data["Depends"]}\n'.encode("utf-8"))
        # file_handle.write(f'License:{data["License"]}\n'.encode("utf-8"))
        # file_handle.write(f'md5:{md5_hash}\n'.encode("utf-8"))
        
        # # 写入多语言描述
        # for key, value in data["Descriptions"].items():
        #     file_handle.write(f'Description\t{key}:'.encode("utf-8") + value + b'\n')
        
        # file_handle.write(f'Description:{data["Description"]}\n\n'.encode("utf-8"))

    def build_windows_exe(self, pip_mirror):
        """
        构建 Windows 可执行文件
        
        使用 Wine 和 PyInstaller 在 Linux 环境下构建 Windows 可执行文件
        需要预先安装 Wine 和 7zip
        
        Returns:
            bool: 构建是否成功
        """
        print("开始构建 Windows 可执行文件...")
        print("此功能需要安装 Wine 和 7zip")
        
        # Wine 安装说明注释
        # print("如果尚未安装 Wine，请参考以下命令:")
        print("# 安装 p7zip-full")
        # print("# apt install p7zip-full")
        if not os.path.isfile("/bin/p7zip"):
            os.system("apt install -y -q p7zip-full")
        if not os.path.isfile("/opt/wine-stable/bin/wine"):
            print("sudo dpkg --add-architecture i386")
            os.system("sudo dpkg --add-architecture i386")
            print("sudo mkdir -pm755 /etc/apt/keyrings")
            os.system("sudo mkdir -pm755 /etc/apt/keyrings")
            print("sudo wget -O /etc/apt/keyrings/winehq-archive.key http://dl.winehq.org/wine-builds/winehq.key")
            os.system("sudo wget -O /etc/apt/keyrings/winehq-archive.key http://dl.winehq.org/wine-builds/winehq.key")
            # print("#")
            # print("# Ubuntu 22.04:")
            print("sudo wget -NP /etc/apt/sources.list.d/ http://dl.winehq.org/wine-builds/ubuntu/dists/jammy/winehq-jammy.sources")
            os.system("sudo wget -NP /etc/apt/sources.list.d/ http://dl.winehq.org/wine-builds/ubuntu/dists/jammy/winehq-jammy.sources")
            # print("#")
            # print("# Ubuntu 20.04 / Linux Mint 20.x:")
            # print("# sudo wget -NP /etc/apt/sources.list.d/ http://dl.winehq.org/wine-builds/ubuntu/dists/focal/winehq-focal.sources")
            # print("#")
            # print("# Ubuntu 18.04 / Linux Mint 19.x:")
            # print("# sudo wget -NP /etc/apt/sources.list.d/ http://dl.winehq.org/wine-builds/ubuntu/dists/bionic/winehq-bionic.sources")
            # print("#")
            # print("# Debian 12 Bookworm:")
            # print("# sudo wget -NP /etc/apt/sources.list.d/ http://dl.winehq.org/wine-builds/debian/dists/bookworm/winehq-bookworm.sources")
            # print("#")
            # print("# Debian 11 Bullseye:")
            # print("# sudo wget -NP /etc/apt/sources.list.d/ http://dl.winehq.org/wine-builds/debian/dists/bullseye/winehq-bullseye.sources")
            # print("#")
            print("# 更新软件包列表并安装 Wine:")
            os.system("sed -i 's/https:\/\//http:\/\//g' /etc/apt/sources.list.d/winehq-jammy.sources")
            os.system("sudo apt update -q")
            os.system("sudo apt install --install-recommends winehq-stable -q -y")
        print("=" * 60)
        
        try:
            # 清理输出目录
            if self.out_dir.exists():
                shutil.rmtree(self.out_dir)
            self._safe_mkdir(self.out_dir)
            
            print("正在下载 Python 环境...")
            # 下载 Python 环境
            download_cmd = "wget http://server.jiang1446.i234.me:3000/jiang/jcm/releases/download/lib/python-3.6.7.7z -nv"
            for i in sys.argv:
                # if i == "-y":
                #     y = True
                if i[:9] == "--mirror=":
                    mirror = i[9:]
                    mirrorrul   = mirror + "/releases/download/"
                    download_cmd = "wget " + mirrorrul + "lib/python-3.6.7.7z -nv"

            if os.system(download_cmd) != 0:
                print("下载 Python 环境失败")
                return False
            
            print("正在解压 Python 环境...")
            # 解压 Python 环境
            extract_cmd = "cd .out && 7z x ../python-3.6.7.7z"
            if os.system(extract_cmd) != 0:
                print("解压 Python 环境失败")
                return False
            
            print("正在安装 Python 依赖包...")
            # 安装依赖包
            pip_commands = [
                "cd .out/Tools/.python/ && wine python.exe -m pip install " + pip_mirror + " pyinstaller",
                "cd .out/Tools/.python/ && wine python.exe -m pip install " + pip_mirror + " requests",
                "cd .out/Tools/.python/ && wine python.exe -m pip install " + pip_mirror + " psutil",
            ]
            
            for i, cmd in enumerate(pip_commands, 1):
                print(f"执行安装命令 {i}/{len(pip_commands)}...")
                if os.system(cmd) != 0:
                    print(f"执行命令失败: {cmd}")
                    return False
            
            print("正在构建可执行文件...")
            # 构建可执行文件
            build_cmd = (
                "cd .out/Tools/.python/ && "
                "cp ../../../Tools/install.py ./ && "
                "wine Scripts/pyinstaller.exe -F --hidden-import requests --hidden-import psutil --hidden-import csv -c --uac-admin install.py && "
                "cp dist/install.exe ../../../.jcm/install/"
            )
            
            if os.system(build_cmd) != 0:
                print("构建可执行文件失败")
                return False
            
            print("Windows 可执行文件构建成功!")
            return True
            
        except Exception as e:
            print(f"构建 Windows 可执行文件时发生错误: {e}")
            return False

    def build_test_packages(self):
        """
        构建测试包
        
        从 Test/ 目录构建测试用的包文件
        """
        print("开始构建测试包...")
        
        test_dir = Path("Test")
        if not test_dir.exists():
            print("未找到 Test 目录")
            return
        
        output_test_dir = Path("outtest")
        self._safe_mkdir(output_test_dir)
        
        test_count = 0
        for test_item in test_dir.iterdir():
            if test_item.is_dir():
                print(f"正在构建测试包: {test_item.name}")
                shutil.make_archive(
                    str(output_test_dir / test_item.name), 
                    "gztar", 
                    base_dir=str(test_item)
                )
                test_count += 1
        
        print(f"测试包构建完成，共构建 {test_count} 个测试包")

    def _cleanup_temp_files(self):
        """
        清理临时文件和目录
        
        删除构建过程中产生的临时文件，包括 __pycache__ 目录
        """
        print("正在清理临时文件...")
        
        # 清理输出目录
        if self.out_dir.exists():
            shutil.rmtree(self.out_dir)
        
        # 递归删除 __pycache__ 目录
        self._remove_pycache_dirs(Path('.'))
        
        print("临时文件清理完成")

    def _remove_pycache_dirs(self, directory):
        """
        递归删除 __pycache__ 目录
        
        Args:
            directory (Path): 要搜索的目录
        """
        try:
            for item in directory.iterdir():
                if item.is_dir():
                    if item.name == "__pycache__":
                        shutil.rmtree(item)

                        print(f"已删除缓存目录: {item}", end='          \r')
                    else:
                        self._remove_pycache_dirs(item)
        except PermissionError:
            print(f"权限不足，无法访问目录: {directory}")
        except Exception as e:
            print(f"清理目录 {directory} 时发生错误: {e}")

    def copy_jcm_files(self):
        """
        复制 JCM 文件到输出目录
        
        将 .jcm 目录的内容复制到指定位置
        """
        print("正在输出 JCM 文件到 .jcm/ 目录")
        
        if not self.jcm_dir.exists():
            print("未找到 .jcm 目录")
            return
        
        try:
            print(f"JCM 文件已输出到: {self.jcm_dir.absolute()}")
        except Exception as e:
            print(f"处理 JCM 文件时发生错误: {e}")

    def show_disk_usage(self):
        """
        显示磁盘使用情况和系统信息
        
        跨平台显示当前分区的磁盘使用情况和系统版本信息
        """
        import platform
        
        print("系统信息:")
        print(f"操作系统: {platform.system()} {platform.release()}")
        print(f"系统版本: {platform.version()}")
        print(f"架构: {platform.machine()}")
        print(f"Python版本: {platform.python_version()}")
        print("-" * 50)
        
        print("当前磁盘使用情况:")
        try:
            # 获取当前工作目录的磁盘使用情况
            current_path = os.getcwd()
            
            if hasattr(shutil, 'disk_usage'):
                # Python 3.3+ 支持
                total, used, free = shutil.disk_usage(current_path)
            else:
                # 兼容旧版本 Python
                import statvfs
                statvfs_result = os.statvfs(current_path)
                total = statvfs_result.f_frsize * statvfs_result.f_blocks
                free = statvfs_result.f_frsize * statvfs_result.f_available
                used = total - free
            
            # 转换为 GB 单位
            total_gb = total / (1024**3)
            used_gb = used / (1024**3)
            free_gb = free / (1024**3)
            used_percent = (used / total) * 100
            
            print(f"当前路径: {current_path}")
            print(f"总容量: {total_gb:.2f} GB")
            print(f"已使用: {used_gb:.2f} GB ({used_percent:.1f}%)")
            print(f"剩余空间: {free_gb:.2f} GB")
            
            # 显示使用情况条形图
            bar_length = 40
            used_bars = int((used_percent / 100) * bar_length)
            free_bars = bar_length - used_bars
            print(f"使用情况: [{'█' * used_bars}{'░' * free_bars}] {used_percent:.1f}%")
            
        except Exception as e:
            print(f"获取磁盘使用情况时发生错误: {e}")
            # 备用方案：尝试使用系统命令
            try:
                if os.name == 'nt':  # Windows
                    os.system('fsutil volume diskfree .')
                else:  # Linux/Unix
                    os.system('df -h .')
            except:
                print("无法获取磁盘使用情况")
        
        print("-" * 50)

def outinstall():
    cp = "cp -rf "
    if not os.path.isdir(".jcm/install"):
        os.mkdir(".jcm/install")
    os.system(cp + "Tools/install.py .jcm/install")

def main():
    """
    主函数 - 处理命令行参数并执行相应操作
    """
    print("JCM 包构建工具启动")
    print("=" * 50)
    
    # 创建包构建器实例
    builder = JCMPackageBuilder()
    
    # 处理命令行参数
    if len(sys.argv) == 1:
        # 默认执行安装操作
        sys.argv.append("install")
    
    i = 1
    pipmirror = ""
    for arg in sys.argv[1:]:
        ii = arg.split("=")
        i = i + 1
        if ii[0] == "--pipmirror":
            pipmirror = " -i " + arg[len(ii[0]) + 1:]
            if pipmirror == " -i http:":
                pipmirror += sys.argv[i]
    for i in sys.argv[1:]:
        ii = i.split("=")
        if ii[0] == "--trusted-host":
            pipmirror += " --trusted-host " + i[len(ii[0]) + 1:]


    # 解析命令行参数
    for arg in sys.argv[1:]:
        if arg == "--exe":
            print("开始构建 Windows 可执行文件...")
            if not builder.build_windows_exe(pipmirror):
                exit(-1)
            
        elif arg == "--test":
            print("开始构建测试包...")
            builder.show_disk_usage()
            builder.build_test_packages()
            
        elif arg == "install" or arg == "build":
            print("开始构建所有包...")
            builder.show_disk_usage()
            builder.build_all_packages()
            builder.copy_jcm_files()
            outinstall()
            
        elif arg[:len("--pipmirror")] == "--pipmirror":
            None
        elif arg[:len("--trusted-host")] == "--trusted-host":
            None
        elif arg[:len("--mirror")] == "--mirror":
            None
        else:
            print(f"未知参数: {arg}")
            print("可用参数:")
            print("  install/build  - 构建所有包")
            print("  --exe         - 构建 Windows 可执行文件")
            print("  --test        - 构建测试包")
    
    print("=" * 50)
    print("JCM 包构建工具执行完成")


if __name__ == "__main__":
    main()