diff --git a/.gitignore b/.gitignore index 5fbd6c8e..41f4e616 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,13 @@ venv/ .venv/ __pycache__ .ipynb_checkpoints/ +build +dist +*.egg-info +*.pyc +*.pyo +*.pyd +*.log +*.bak +*.swp +.DS_Store diff --git a/pyproject.toml b/pyproject.toml index 1fe2ba91..ab09c077 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ version_file = "src/c3/_version.py" [project] name = "claimed" -version = "0.2.3" +version = "0.2.6" authors = [ { name="The CLAIMED authors", email="claimed-framework@proton.me"}, ] @@ -31,6 +31,7 @@ dependencies = [ 'traitlets >= 5.11.2', 'pandas', 'boto3', + 's3fs', ] [project.urls] diff --git a/src/claimed/claimed.py b/src/claimed/claimed.py index c19c11b1..767aebed 100644 --- a/src/claimed/claimed.py +++ b/src/claimed/claimed.py @@ -1,12 +1,119 @@ +import importlib +import inspect +import os import subprocess import sys -import os + + +def _parse_kwargs(rest, sig): + """Parse --key value pairs from a list of CLI tokens, coerce types via signature.""" + kwargs = {} + i = 0 + while i < len(rest): + token = rest[i] + if token.startswith('--'): + key = token[2:].replace('-', '_') + if i + 1 < len(rest) and not rest[i + 1].startswith('--'): + kwargs[key] = rest[i + 1] + i += 2 + else: + # bare flag → True + kwargs[key] = True + i += 1 + else: + i += 1 + + # Coerce string values to the expected type using annotation or default + for name, param in sig.parameters.items(): + if name not in kwargs: + continue + val = kwargs[name] + if not isinstance(val, str): + continue + # Try annotation first + ann = param.annotation + if ann is not inspect.Parameter.empty: + target = ann + if hasattr(ann, '__origin__'): + # e.g. Optional[X] – skip complex generics + continue + try: + kwargs[name] = target(val) + continue + except Exception: + pass + # Fall back to the type of the default value + default = param.default + if default is not inspect.Parameter.empty and default is not None: + try: + kwargs[name] = type(default)(val) + except Exception: + pass + + return kwargs + + +def _run_module(args): + if not args: + print("Usage: claimed run [--param-name value ...] [--help]") + sys.exit(1) + + module_path = args[0] + rest = args[1:] + + # Import the target module + try: + module = importlib.import_module(module_path) + except ImportError as e: + print(f"Error: cannot import module '{module_path}': {e}") + sys.exit(1) + + if not hasattr(module, 'run'): + print(f"Error: module '{module_path}' has no 'run' function.") + sys.exit(1) + + fn = module.run + sig = inspect.signature(fn) + + # --help: print signature and docstring + if '--help' in rest: + print(f"Module : {module_path}") + print(f"Function: {module_path}.run{sig}") + doc = inspect.getdoc(fn) + if doc: + print(f"\n{doc}\n") + print("Parameters:") + for pname, param in sig.parameters.items(): + flag = '--' + pname.replace('_', '-') + ann = param.annotation + type_hint = ( + ann.__name__ if (ann is not inspect.Parameter.empty and hasattr(ann, '__name__')) + else str(ann) if ann is not inspect.Parameter.empty + else 'any' + ) + default = ( + f' (default: {param.default!r})' + if param.default is not inspect.Parameter.empty + else ' (required)' + ) + print(f" {flag} <{type_hint}>{default}") + sys.exit(0) + + kwargs = _parse_kwargs(rest, sig) + fn(**kwargs) def main(): + if len(sys.argv) > 1 and sys.argv[1] == 'run': + _run_module(sys.argv[2:]) + return + dir_path = os.path.dirname(os.path.realpath(__file__)) - return subprocess.call(f'{dir_path}/scripts/claimed ' + ' '.join(sys.argv[1:]), shell=True) + return subprocess.call( + f'{dir_path}/scripts/claimed ' + ' '.join(sys.argv[1:]), shell=True + ) if __name__ == '__main__': main() + diff --git a/src/claimed/components/util/cosutils.py b/src/claimed/components/util/cosutils.py index 758341f2..9181e16d 100644 --- a/src/claimed/components/util/cosutils.py +++ b/src/claimed/components/util/cosutils.py @@ -23,7 +23,7 @@ import s3fs import sys import glob -from c3.operator_utils import explode_connection_string +from claimed.c3.operator_utils import explode_connection_string # In[ ]: @@ -47,7 +47,23 @@ # In[ ]: -def run(cos_connection, local_path, operation, recursive = False, log_level = logging.INFO): +def run( + cos_connection: str, + local_path: str, + operation: str, + recursive: bool = False, + log_level: str = 'INFO', +) -> None: + """ + Perform a COS/S3 file operation. + + cos_connection: s3://access_key_id:secret_access_key@endpoint/bucket/path + operation: one of mkdir | ls | find | get | put | rm | sync_to_cos | sync_to_local | glob + local_path: local file or directory used for get / put / sync operations + recursive: apply the operation recursively + log_level: logging verbosity: DEBUG | INFO | WARNING | ERROR (default: INFO) + """ + logging.basicConfig(level=getattr(logging, log_level.upper(), logging.INFO)) (access_key_id, secret_access_key, endpoint, cos_path) = explode_connection_string(cos_connection) s3 = s3fs.S3FileSystem( anon=False,