From b9a2f5abc60fe19181cd96fcefdcbf6c1d1f652e Mon Sep 17 00:00:00 2001 From: Kyle Bowman Date: Sat, 9 Mar 2024 12:38:11 -0500 Subject: [PATCH] Add initial command functionality. * Command object interface defined * define, parse, run flow works in general * command decorator defined * get_parser supports int, float, str * command.run() supports CLI args or args supplied in function call * testing added for major functionality --- design.md | 76 ++++++++++++++++++------ findings.md | 32 +++++++++- src/proto/prototype.py | 70 ++++++++++++---------- src/proto/utils.py | 69 +++++++++++++++++----- tests/test_proto.py | 52 +++++++++++++---- tests/test_utils.py | 130 +++++++++++++++++++++++++++++++++++------ 6 files changed, 336 insertions(+), 93 deletions(-) diff --git a/design.md b/design.md index d5fa954..db8a470 100644 --- a/design.md +++ b/design.md @@ -71,7 +71,34 @@ subparser = make_subparser(cmd) register_subparser(parser, subparser) ``` -### Parser Dispatch: Parsing and Dispatching +### Parser Construction: Defining Parse options +Command line arguments are always a string. You need to tell `argparse` how to +interpret that string. At minimum, what type does an argument indicate. You +might also need to specify *how* to construct that type from a string. + +* Scalar types should be straight forward. + * Bake those in. Permit override. +* Users might want to write custom logic. + * Make it easy to wire up string->type constructor. + +This seems like a choice opportunity to use `@singledispatch`: +* function accepts a Parameter and returns a dict defining the argument specs +* function behavior depends on a single switch - Parameter type. +* enables users to register alternate implementations without cracking into core code + +Here are some of the types that I haven't implemented that seem useful: +* `bool` - argparse discourages use of bool directly (using `type`), but you can look into the `store_true`, `store_false` actions. +* `pathlib.Path` +* `Enum`s - might invoke `choices` for that argument. +* `Union`s, - might iterate through Union constructors and prioritize first encountered? +* `list`, `tuple`, `range` `GenericAlias` (e.g. `list[int]`) - maybe use `nargs` in `parser.add_arguments`. +* `dict` - user defined; from_json; possibly define in CLI itself; +* `bytes`, `bytearray`, `memoryview` - user-defined +* `set`, `frozenset` - user-defined +* modules, functions, methods, other... - custom + + +### Parser Dispatch: Invoking Parse and Dispath * Invoke `MainCLI` methods. ``` python if __name__ == "__main__": @@ -88,10 +115,13 @@ if __name__ == "__main__": # Possible Enhancements -## (?) Distribute a CLI testing library -* Shell out? -* Mock sys.argv? -* Don't bother? +## Command aliases + +```python +@command(aliases=['alias']) +def long_name(*args, **kwargs): + pass +``` ## Validation Ideas: 1. Define a predicate data type: Callable[Any, bool] @@ -111,20 +141,15 @@ def is_valid(arg1, arg2): return True if X else False ``` -## (?) Aggregate Commands - We need a way to fetch Cmds from their modules and register with CLI. - Like `instances = fetch_subclasses(Cmd)` - Make generic? fetch(subclasses, Cmd) +## (?) Make get_parser() more flexible +See also the section about Help documentation. I'm not sure what must be done +when invoking `parser.add_argument()` and what can be done post construction. +If something must be done during `add_argument()` is there a way to remove +then re-add an argument? -## (?) Serializing Ideas - cmd.to_json() - packs command and args into json - add a to_json Cmd? - add by default to MainCLI() parser? - - => Mirror with from_json ## (?) Help documentation -This dealas with two things: +This deals with two things: * How to manually specify command help? * How to automatically generate some kind of help? @@ -138,6 +163,23 @@ Idea One: Define as dictionary and pass unpacked to @command. Idea Two: Define helper functions around idea Zero +## (?) Distribute a CLI testing library +* Mock sys.argv? (Use a context manager?) +* Shell out? +* Promote making useful test functions available for REPL based development. + +## (?) Aggregate Commands + We need a way to fetch Cmds from their modules and register with CLI. + Like `instances = fetch_subclasses(Cmd)` + Make generic? fetch(subclasses, Cmd) + +## (?) Serializing Ideas + cmd.to_json() - packs command and args into json + add a to_json Cmd? + add by default to MainCLI() parser? + + => Mirror with from_json + ## (?) Autoimport flexibility Enhancement to MainCLI - add a param that enables users to define -the command search space.jK:W \ No newline at end of file +the command search space. \ No newline at end of file diff --git a/findings.md b/findings.md index a8414a5..ef19191 100644 --- a/findings.md +++ b/findings.md @@ -1,11 +1,36 @@ +# Mental Stack +Usage: +* Decorate an annotated function +* Use fn.parse() +* use fn.run +* For testing, haphazardly use sys.argv. + +## Comment Keywords +Don't commit anything that starts with "temporarily". + +Search by using `git grep "# [A-Z]*:"`. + +* TODO - temporarily bookmark a TBD task +* BUG - temporarily bookmark something that needs to be fixed and tested. +* NOTE - describe an important consideration about the code. May be a design consideration, reminder, etc.. +* ASSUME - describes an assumption that must be true for the code to behave as intended. +* HACK - describes a non-obvious solution; typically indicates a "misuse" of another bit of code +* QUESTION - temporarily indicates something that should be further researched + # Findings from experiments ## About Parsers * Subparsers are of same type as parser (`argparse.ArgumentParser`) * Corollary: You can nest subparsers indefinitely +* ~~The `argparse.Namespace` object always stores values as strings.~~ +* Unless you explictly used `type` in `add_argument`, a CLI argument is parsed as a string. +* The `--` is stripped from optional flags when parsed into an `argparse.Namespace` +* An argument is stored as an `Action`. + * Try to reach into the `parser._actions` list to set attributes between `add_argument` and `parser_args`. + * For reference, `parser.add_arguments` seems to be defined in `argparse._ActionsContainer.add_argments` ## About Decorators -* You can define @instance.method wrappers +* You can define `@instance.method` wrappers * This might be useful to do lower level argparse stuff, like setting help info. Example: @@ -16,7 +41,7 @@ def wrap(self, fn: Callable, *args, **kwargs): return str(fn(*args, **kwargs)).upper() ``` -You can enhance the function decorate the functino `new_name` : +You can enhance the function decorate the function `new_name` : ``` python @test.wrap def new_name(): @@ -29,6 +54,9 @@ if __name__ == "__main__": print(new_name) # "KYLE" ``` +## About Signatures +* Argnames = Signature - Kwargs; this is a good way to report missing required arg + # About MainCLI Purpose: 1. Search for other commands to register with it diff --git a/src/proto/prototype.py b/src/proto/prototype.py index 278bd4e..27609bb 100644 --- a/src/proto/prototype.py +++ b/src/proto/prototype.py @@ -1,37 +1,47 @@ -import argparse -from typing import Optional, Callable -from proto.utils import get_defaults, get_types +from argparse import ArgumentError +from collections.abc import Callable +from functools import partial +from typing import Optional +from proto.utils import get_defaults, get_types, get_parser + -# NOTE: Currently, Command has no way to specify fn arguments. -# NOTE: Args for a command are going to come from CLI. -# Which means there will be a parser.parse_arguments() call. -# NOTE: To test a Command whose function takes arguments, we -# will need to mock parser.parse_arguments. class Command: def __init__(self, fn: Callable): - self.name = fn.__name__ - # self.types = get_types(fn) - # self.defaults = get_defaults(fn) - # (Property?) self.run = fn - # self.parser = get_parser(fn) + self.name = fn.__name__ + self._run = fn + self.types = get_types(fn) + self.parser = get_parser(fn) + self.required, self.defaults = get_defaults(fn) + self.args = [] + self.kwargs = {} - @property - def parser(self)-> argparse.ArgumentParser: - pass + def parse(self, args: Optional[list[str]] = None): + """ Parse the list to populate args and kwargs attributes. + If no list is specified, sys.argv is used. """ + # NOTE: Make parsing idemptotent. + self.args = [] + self.kwargs={} - def run(self): - pass + options = vars(self.parser.parse_args(args=args)) + for k, v in options.items(): + if k in self.required.keys(): + self.args.append(v) + elif k in self.defaults.keys(): + self.kwargs[k] = v + else: + # NOTE: Unrecognized args should be caught earlier in parse_args(). + raise ArgumentError(k,"Unrecognized argument {k}") + return self.args, self.kwargs -def command(fn: Callable, aliases: list = [], help: Optional[str] = None): - return Command(fn) + def run(self, *args, **kwargs): + """ Invokes function using args/kwargs if specified. Otherwise, uses self. """ + if not args and not kwargs: + return partial(self._run, *self.args, **self.kwargs)() + else: + args = args if args else self.args + kwargs = kwargs if kwargs else self.kwargs + return self._run(*args, **kwargs) -#class MainCLI(): -# -# # Signature? -# def __init__(self): -# # self.parser = # default parser -# # self.commands = get_commands() -# pass -# -# def parse(self)->Command: -# pass \ No newline at end of file +def command(fn: Callable): + """ Defines a decorator that is used to turn a function into a Command. """ + return Command(fn) \ No newline at end of file diff --git a/src/proto/utils.py b/src/proto/utils.py index a73a031..2116ea9 100644 --- a/src/proto/utils.py +++ b/src/proto/utils.py @@ -1,30 +1,69 @@ -from typing import Callable +from collections.abc import Callable import inspect +import argparse -def get_defaults(fn: Callable)->dict: - """ Returns a dictionary of parameter names and defaults. """ +def get_defaults(fn: Callable)->tuple[list, dict]: + """ Returns a dict of required arguments and a dict of kwarg defaults. """ sig = inspect.signature(fn) - defaults = {} + args = {} + kwargs = {} for name, prm in sig.parameters.items(): - # ASSUME: __init__ is defined with "self". It's a Pythonic standard to use self, but it's convention, not rule. Beware. - if name == "self": + # ASSUME: It's a Pythonic standard to use self and cls, but it's convention, not rule. Beware. + if name == "self" or name == "cls": continue - if prm.default == inspect._empty: - defaults[name] = None + if prm.default == inspect._empty: + # NOTE: Don't use args[name]=None. None could be a valid argument + # and we want to flag that a required argument is missing a value. + args[name] = inspect._empty else: - defaults[name] = prm.default - return defaults + kwargs[name] = prm.default + return args, kwargs -def get_types(fn: Callable)->dict[type]: - """ Returns a dictionary of parameter names and types. """ +def get_types(fn: Callable)->dict[str]: + """ Returns a dictionary of parameter names and their types as strings. """ sig = inspect.signature(fn) types = {} for name, prm in sig.parameters.items(): - # ASSUME: __init__ is defined with "self". It's a Pythonic standard to use self, but it's convention, not rule. Beware. - if name == "self": + # ASSUME: It's a Pythonic standard to use self, but it's convention, not rule. Beware. + if name == "self" or name == "cls": continue if prm.annotation == inspect._empty: types[name] = None else: types[name] = prm.annotation - return types \ No newline at end of file + return types + +def get_parser(fn: Callable)->argparse.ArgumentParser: + """ Returns an argparse.ArgumentParser based on the function's signature. """ + sig = inspect.signature(fn) + parser = argparse.ArgumentParser() + + for prm in sig.parameters.values(): + argname = None + arg_specs = dict() + + # ASSUME: It's a Pythonic standard to use self, but it's convention, not rule. Beware. + if prm.name == "self" or prm.name == "cls": + continue + + # ASSUME: If a function specifies a default argument, we tell the parser to consider it optional. + if prm.default != prm.empty: + # NOTE: Argparse requires optional args to start with '-' or '--'. + # Doc reference: https://docs.python.org/3.11/library/argparse.html#id5 + argname = "--" + prm.name + arg_specs['required'] = False + arg_specs['default']=prm.default + else: + argname = prm.name + + # NOTE: If you don't specify type in add_argument(), it will be parsed as a string. + # HACK: Whenever you see you can use it as an initializer. + # It works consistently, but I haven't seen it as defined/supported behavior. + # Ex: type(42)('36') creates an integer 36. + if prm.annotation in (int, float, str): + arg_specs['type'] = prm.annotation + else: + pass + + parser.add_argument(argname, **arg_specs) + return parser diff --git a/tests/test_proto.py b/tests/test_proto.py index 7c3e2ea..fe16c7f 100644 --- a/tests/test_proto.py +++ b/tests/test_proto.py @@ -1,24 +1,54 @@ +from typing import Protocol import pytest from proto import command, Command -from typing import Protocol - class Stringable(Protocol): def __str__(self)->str: ... -def echo(arg: Stringable)->str: - return "Command" + str(arg) +def echo_fn(arg: Stringable)->str: + return str(arg) @command -def echo_decorated(arg: Stringable)->str: - return "@command" + str(arg) +def echo(arg: Stringable)->str: + return echo_fn(arg) def test_attributes(): - cmd = Command(echo) - assert cmd.name == "echo" + cmd = Command(echo_fn) + assert cmd.name == "echo_fn" + cmd2 = echo + assert cmd2.name == "echo" + +def test_parse(): + echo.parse(args=['42']) + assert echo.kwargs == {} + # Okay for 42 to be a string because echo is defined on stringables + assert echo.args[0] == '42' + +def test_parse_argument_error(): + with pytest.raises(SystemExit): + echo.parse(args=['echo_target','--unknown']) + +def test_parse_idempotency(): + echo.parse(args=['42']) + echo.parse(args=['something else']) + assert '42' not in echo.args + assert 'something else' in echo.args + +def test_run_parsed_args(): + echo.parse(args=['yippee!']) + assert "yippee!" in echo.run() + +def test_run_override_parsed(): + echo.parse(args=['yippee!']) + assert "override" in echo.run("override") + assert '42' in echo.run(42) + -def test_attributes_2(): - cmd = echo_decorated - assert cmd.name == "echo_decorated" \ No newline at end of file +if __name__ == "__main__": + @command + def test(char: str, n: int = 42)->tuple[str,int]: + return char, n + + test.parse(args=['--n', '36', 'char_field']) \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 1610bf9..a7c4054 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,51 +1,145 @@ +import inspect +import sys from typing import Optional -import proto.utils +import pytest +from proto.utils import get_defaults, get_types, get_parser ############## # Begin Data # -- signature testing ############## -def fn(string, num)->str: +# TODO: (?) Move these dummy functions to their own module within test dir. +class DummyCallable: + + def __call__(self, string: str = "default", num: int=42): + return f"String: {string} \n Integer: {num} \n" + +class DummyClass: + + def dummy_method(self, string: str = "default", num: int=42): + return f"String: {string} \n Integer: {num} \n" + + @classmethod + def dummy_classmethod(cls, string: str = "default", num: int=42): + return f"String: {string} \n Integer: {num} \n" + +def dummy_fn_no_signature(string, num)->str: return f"String: {string} \n Integer: {num} \n" -def fn_annotated(string: str, num: int)->str: +def dummy_fn_typed(string: str, num: int)->str: return f"String: {string} \n Integer: {num} \n" -def fn_optional(string: Optional[str], num: Optional[int] = 42)->str: +def dummy_fn_optional(string: Optional[str], num: Optional[int] = 42)->str: return f"String: {string} \n Integer: {num} \n" -def fn_default(string: str="spam", num: int = 42)->str: +# NOTE: The dummy_fn_full function is the prototypical "happy path" for signagures. +def dummy_fn_full(string: str="default", num: int = 42)->str: return f"String: {string} \n Integer: {num} \n" ############### # Begin Tests # ############### + +def test_get_default_equivalence(): + """ Ensures that defaults are treated the same amongst Callables. """ + callable_defaults = get_defaults(DummyCallable()) + method_defaults = get_defaults(DummyClass().dummy_method) + function_defaults = get_defaults(dummy_fn_full) + assert callable_defaults == method_defaults == function_defaults + +def test_get_types_equivalence(): + """ Ensures that type signatures are treated the same amongst Callables. """ + callable_types = get_types(DummyCallable()) + method_types = get_types(DummyClass().dummy_method) + function_types = get_types(dummy_fn_full) + assert callable_types == method_types == function_types + +@pytest.mark.skip("Equivalence is not defined for argparse.ArgumentParser") +def test_get_parser_equivalence(): + callable_parser = get_parser(DummyCallable()) + method_parser = get_parser(DummyClass().dummy_method) + function_parser = get_parser(dummy_fn_full) + assert callable_parser == method_parser == function_parser + +# ASSUME: Assuming the above tests are valid and pass, we only have to test one +# kind of the Callable and the results should hold for the others. def test_get_defaults(): - defaults = proto.utils.get_defaults(fn_default) - assert defaults['string'] == "spam" - assert defaults['num'] == 42 + args, kwargs = get_defaults(dummy_fn_full) + assert args == {} + assert kwargs['string'] == "default" + assert kwargs['num'] == 42 def test_get_defaults_unspecified(): - defaults = proto.utils.get_defaults(fn) - assert defaults['string'] is None - assert defaults['num'] is None + args, kwargs = get_defaults(dummy_fn_no_signature) + assert args == {'string': inspect._empty, 'num': inspect._empty} + assert kwargs == {} def test_get_types(): - types = proto.utils.get_types(fn_annotated) + types = get_types(dummy_fn_typed) assert types['string'] == str assert types['num'] == int def test_get_types_unspecified(): - types = proto.utils.get_types(fn) + types = get_types(dummy_fn_no_signature) assert types['string'] is None assert types['num'] is None def test_get_types_optional(): - types = proto.utils.get_types(fn_optional) + types = get_types(dummy_fn_optional) assert types['string'] == Optional[str] assert types['num'] == Optional[int] def test_validating_optional(): - types = proto.utils.get_types(fn_optional) - defaults = proto.utils.get_defaults(fn_optional) - assert isinstance(defaults['string'],types['string']) - assert isinstance(defaults['num'],types['num']) + types = get_types(dummy_fn_optional) + _, kwargs = get_defaults(dummy_fn_optional) + assert isinstance(kwargs['num'],types['num']) + +# NOTE: The parser.parse_args() method always returns keys and values as strings. +# You must cast values yourself to compare. +# NOTE: sys.argv[0] is the program name and is not needed. +def test_get_parser_defaults(): + """ If a fn default is specified, use keyword syntax (optional). """ + string="not default" + num=42 + sys.argv[1:] = [f"--string={string}", "--num", str(num)] + parser=get_parser(dummy_fn_full) + args = parser.parse_args() + assert args.string == string + assert int(args.num) == num + +def test_get_parser_no_defaults(): + """ If no function default is specified, use postional syntax (required). """ + string="not default" + num=42 + sys.argv[1:] = [string, str(num)] + parser=get_parser(dummy_fn_no_signature) + args = parser.parse_args() + assert args.string == string + +def test_get_parser_method(): + """ Ensures that 'self' and 'cls' are ignored. """ + string="not default" + num=42 + sys.argv[1:] = [f"--string={string}", "--num", str(num)] + parser=get_parser(DummyClass().dummy_method) + args = parser.parse_args() + assert args.string == string + parser2=get_parser(DummyClass.dummy_classmethod) + args2 = parser2.parse_args() + assert args2.string == string + +def test_get_parser_types_scalar(): + """ Asserts ints, strings, and floats, are parsed from CLI as types. """ + string="not default" + num="42" + sys.argv[1:] = [f"--string={string}", "--num", str(num)] + parser=get_parser(dummy_fn_full) + args = parser.parse_args() + assert args.string == string + assert args.num == int(num) + + +if __name__ == "__main__": + import inspect + sig = inspect.signature(dummy_fn_full) + p = sig.parameters['string'] + p2 = sig.parameters['num'] \ No newline at end of file -- 2.39.5