Returns the account ID as a string associated with the acct object.
def acct_id(self, acct):
+ """Returns the account ID as a string associated with the `acct` object."""
+ raise NotImplementedError
def attributes(self)
Returns a dict of all metadata attribute names and values.
def attributes(self):
+ """Returns a dict of all metadata attribute names and values."""
+ raise NotImplementedError
def accounts(self, acct_ids=None, include=None, exclude=None)
@@ -1403,6 +1418,21 @@ Methods
Duplicate account IDs do not result in duplicate account objects. To
filter the list based on account metadata, include and exclude dicts
can be specified identifying the keys and matching values.
+
+
+Expand source code
+
+def accounts(self, acct_ids=None, include=None, exclude=None):
+ """Returns a list of account objects representing accounts and metadata.
+
+ Without any arguments, all accounts are returned. A list of `acct_ids`
+ may be provided to limit the list to accounts matching the account IDs.
+ Duplicate account IDs do not result in duplicate account objects. To
+ filter the list based on account metadata, `include` and `exclude` dicts
+ can be specified identifying the keys and matching values.
+ """
+ raise NotImplementedError
+
@@ -1496,6 +1526,18 @@ Methods
Returns the account ID associated with the acct object.
This is an identity function as the account ID of an account object
loaded by this class is the account object itself, thus acct returned.
+Expand source code
+
+def acct_id(self, acct):
+ """Returns the account ID associated with the `acct` object.
+
+ This is an identity function as the account ID of an account object
+ loaded by this class is the account object itself, thus `acct` returned.
+ """
+ return acct
+
def attributes(self)
@@ -1505,6 +1547,19 @@ Methods
IdentityAccountLoader builds account objects that are simply strings
representing the ID of an account. Because there is no other metadata
associated with these objects, this method returns an empty dict.
+
+
+Expand source code
+
+def attributes(self):
+ """Returns a dict of all metadata attribute names and values.
+
+ `IdentityAccountLoader` builds account objects that are simply strings
+ representing the ID of an account. Because there is no other metadata
+ associated with these objects, this method returns an empty dict.
+ """
+ return {}
+
def accounts(self, acct_ids=None, include=None, exclude=None)
@@ -1517,6 +1572,28 @@ Methods
objects without duplicates. It is an identity function. Because there
are no loaded accounts, use of include and exclude parameters will
raise an AttributeError.
+
+
+Expand source code
+
+def accounts(self, acct_ids=None, include=None, exclude=None):
+ """Returns a list of account objects.
+
+ Without any arguments, no accounts are returned because this class does
+ not load a list of accounts from an external data source. Instead, this
+ method returns the same list of `acct_ids` as the list of account
+ objects without duplicates. It is an identity function. Because there
+ are no loaded accounts, use of `include` and `exclude` parameters will
+ raise an AttributeError.
+ """
+ if include or exclude:
+ raise AttributeError("Cannot use filters as no attributes are defined")
+
+ if acct_ids is None:
+ return []
+
+ return list(set(acct_ids))
+
@@ -2143,6 +2220,20 @@ Methods
instance of this MetaAccountLoader via MetaAccountLoader.accounts().
This method returns the string representing the account ID of the
account object.
+
+
+Expand source code
+
+def acct_id(self, acct):
+ """Returns the account ID associated with the `acct` object.
+
+ The `acct` parameter must be an account object that was created by an
+ instance of this `MetaAccountLoader` via `MetaAccountLoader.accounts`.
+ This method returns the string representing the account ID of the
+ account object.
+ """
+ return getattr(acct, self.id_attr)
+
def attributes(self)
@@ -2165,6 +2256,36 @@ Methods
assert attrs['status'] == {'active', 'suspended'}
assert attrs['id'] == {'100200300400', '200300400100', '300400100200'}
+
+
+Expand source code
+
+def attributes(self):
+ """Returns a dict of all metadata attribute names and values.
+
+ This method returns the metadata associated with the account objects
+ created by this instance of `MetaAccountLoader`. The keys of the
+ returned dict are the attribute names attached to the account objects.
+ The value of each key is a set representing all of the possible values
+ assigned to that attribute. For example:
+
+ accts = [
+ {'id': '100200300400', 'env': 'prod', 'status': 'active'},
+ {'id': '200300400100', 'env': 'prod', 'status': 'suspended'},
+ {'id': '300400100200', 'env': 'dev', 'status': 'active'},
+ ]
+ loader = acctload.MetaAccountLoader(accts)
+ attrs = loader.attributes()
+ assert attrs['env'] == {'prod', 'dev'}
+ assert attrs['status'] == {'active', 'suspended'}
+ assert attrs['id'] == {'100200300400', '200300400100', '300400100200'}
+ """
+ d = defaultdict(set)
+ for acct in self.accts:
+ for attr, value in acct.items():
+ d[attr].add(value)
+ return d
+
def accounts(self, acct_ids=None, include=None, exclude=None)
@@ -2209,6 +2330,84 @@ Methods
to an invalid attribute name, AttributeError is raised. Finally, if
include is not set, then all accounts are matched. Likewise, if
exclude is not set, then no accounts are excluded.
+
+
+Expand source code
+
+def accounts(self, acct_ids=None, include=None, exclude=None):
+ """Returns a list of account objects.
+
+ Without any arguments, account objects for all accounts are returned. A
+ list of `acct_ids` may be provided to limit the list to accounts
+ matching the account IDs. If one or more specified account IDs is
+ missing, `AccountsNotFoundError` is raised.
+
+ The returned list of accounts can be filtered further by providing dicts
+ for `include` and `exclude` parameters that specify attributes and a
+ list of values for those attributes that must match. For example,
+ assuming the following accounts:
+
+ accts = [
+ {'id': '100200300400', 'env': 'prod', 'status': 'active'},
+ {'id': '200300400100', 'env': 'prod', 'status': 'suspended'},
+ {'id': '300400100200', 'env': 'dev', 'status': 'active'},
+ ]
+ loader = acctload.MetaAccountLoader(accts)
+
+ To filter active accounts, use the following:
+
+ include = {'status': ['active']}
+
+ To filter active *and* production accounts:
+
+ include = {'status': ['active'], 'env': ['prod']}
+
+ To filter production accounts, but not suspended:
+
+ include = {'env': ['prod']}
+ exclude = {'status': ['suspended']}
+
+ To filter active *or* suspended accounts, but not dev accounts:
+
+ include = {'status': ['active', 'suspended']}
+ exclude = {'env': ['dev']}
+
+ Both `include` and `exclude` filters are applied after the initial
+ account list has been determined, which is all accounts unless it was
+ first limited by `acct_ids`. Then, the `include` filter is applied,
+ followed by the `exclude` filter. If a filter dict has multiple keys,
+ then *each* key must match. A key matches if at least *one* of the
+ values matches as illustrated in the examples above. If a filter refers
+ to an invalid attribute name, `AttributeError` is raised. Finally, if
+ `include` is not set, then all accounts are matched. Likewise, if
+ `exclude` is not set, then no accounts are excluded.
+ """
+ accts = self.accts
+ acct_ids = [] if acct_ids is None else acct_ids
+ include = {} if include is None else include
+ exclude = {} if exclude is None else exclude
+
+ # Limit our account list to the requested IDs
+ if acct_ids:
+ requested = set(acct_ids)
+ all_ids = (a[self.id_attr] for a in self.accts)
+
+ missing_acct_ids = requested.difference(all_ids)
+ if missing_acct_ids:
+ raise AccountsNotFoundError(list(missing_acct_ids))
+
+ accts = (a for a in self.accts if a[self.id_attr] in requested)
+
+ # Make sure the filters contain valid attribute names
+ for attr in itertools.chain(include.keys(), exclude.keys()):
+ if attr not in self.attrs:
+ raise AttributeError(f"Invalid attribute '{attr}' in filter")
+
+ # Limit our account list by the user-supplied filters
+ return [
+ self.CustomAccount(a) for a in accts if self._filter(a, include, exclude)
+ ]
+
@@ -2956,7 +3155,6 @@ Ancestors
no_verify=False,
cache_path=None,
):
-
session = requests.Session()
session.mount("file://", FileAdapter())
@@ -3075,7 +3273,7 @@ Inherited members
def __repr__(self):
pairs = (f"{k}={repr(v)}" for k, v in self._attrs.items())
- return f'Account({", ".join(pairs)})'
+ return f"Account({', '.join(pairs)})"
def __str__(self):
if not self._str_template:
@@ -3106,7 +3304,7 @@ Inherited members
def __init__(self, missing_acct_ids):
self.missing_acct_ids = missing_acct_ids
- super().__init__(f'Account IDs not found: {", ".join(missing_acct_ids)}')
+ super().__init__(f"Account IDs not found: {', '.join(missing_acct_ids)}")
Ancestors
@@ -3238,7 +3436,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/argparse.html b/docs/argparse.html
index 97c404e..35ff6e5 100644
--- a/docs/argparse.html
+++ b/docs/argparse.html
@@ -3,13 +3,13 @@
-
+
awsrun.argparse API documentation
-
+
@@ -291,6 +291,42 @@ Functions
>>> [f(s) for s in ['yes', 'no', 'true', 'false']]
[True, False, True, False]
+
+
+Expand source code
+
+def from_str_to(type_):
+ """Return a cast function to convert a string to a builtin type.
+
+ The `type` parameter is the name of the type as a string. Returns the
+ builtin Python cast function if `type` is "str", "int", or "float". If
+ `type` is "bool", the returned cast function will return `True` for the
+ values "y", "yes", "true", and "1" (case insensitive), otherwise it will
+ return `False`. For any other `type` specified, the builtin `str`
+ function is returned.
+
+ >>> f = from_str_to("str")
+ >>> f("hello")
+ "hello"
+
+ >>> f = from_str_to("int")
+ >>> f("10")
+ 10
+
+ >>> f = from_str_to("float")
+ >>> f("10")
+ 10.0
+
+ >>> f = from_str_to("bool")
+ >>> [f(s) for s in ['yes', 'no', 'true', 'false']]
+ [True, False, True, False]
+ """
+ if type_ in ("str", "int", "float"):
+ return getattr(builtins, type_)
+ if type_ == "bool":
+ return lambda s: s.lower() in ("y", "yes", "true", "True", "1")
+ return str
+
@@ -397,7 +433,7 @@ Ancestors
class AppendAttributeValuePair
-(option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None)
+(option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None, deprecated=False)
-
Argparse action to construct a dict of key/value pairs.
@@ -573,7 +609,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/cache.html b/docs/cache.html
index 5434733..a14b490 100644
--- a/docs/cache.html
+++ b/docs/cache.html
@@ -3,13 +3,13 @@
-
+
awsrun.cache API documentation
-
+
@@ -393,6 +393,28 @@ Methods
invocations of this method will return the cached value until it
expires. If you set refresh parameter to True, the value will be
refreshed and the expiration will be reset before being returned.
+Expand source code
+
+def value(self, refresh=False):
+ """Returns the value.
+
+ The first time this method is called, the value will be obtained by
+ calling the `refresh_fn` supplied in the constructor. Subsequent
+ invocations of this method will return the cached value until it
+ expires. If you set `refresh` parameter to `True`, the value will be
+ refreshed and the expiration will be reset before being returned.
+ """
+ with self._lock:
+ if not refresh and not self.is_expired():
+ return self.load()
+
+ value = self._refresh_fn()
+ self.save(value)
+ LOG.info("refreshed data and saved in cache")
+ return value
+
def is_expired(self)
@@ -403,6 +425,20 @@ Methods
AbstractExpiringValue.value(), the refresh_fn will be called, followed
by save, to renew the cached value. When this returns False, load
is invoked instead to return the value from the cache.
+
+
+Expand source code
+
+def is_expired(self):
+ """Returns `True` if the value needs to be refreshed, `False` otherwise.
+
+ If this returns `True` during the invocation of
+ `AbstractExpiringValue.value`, the `refresh_fn` will be called, followed
+ by `save`, to renew the cached value. When this returns `False`, `load`
+ is invoked instead to return the value from the cache.
+ """
+ raise NotImplementedError
+
def load(self)
@@ -412,6 +448,19 @@ Methods
If is_expired returns False during the invocation of
AbstractExpiringValue.value(), this method is invoked to return the
value from the cache.
+
+
+Expand source code
+
+def load(self):
+ """Returns the value from the cache.
+
+ If `is_expired` returns `False` during the invocation of
+ `AbstractExpiringValue.value`, this method is invoked to return the
+ value from the cache.
+ """
+ raise NotImplementedError
+
def save(self, value)
@@ -421,6 +470,19 @@ Methods
If is_expired returns True during the invocation of
AbstractExpiringValue.value(), this method is invoked to save the new
value to the cache.
+
+
+Expand source code
+
+def save(self, value):
+ """Saves the value to the cache.
+
+ If `is_expired` returns `True` during the invocation of
+ `AbstractExpiringValue.value`, this method is invoked to save the new
+ value to the cache.
+ """
+ raise NotImplementedError
+
@@ -613,7 +675,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/cli.html b/docs/cli.html
index 89b7031..48b7441 100644
--- a/docs/cli.html
+++ b/docs/cli.html
@@ -3,13 +3,13 @@
-
+
awsrun.cli API documentation
-
+
@@ -1815,7 +1815,7 @@ Troubleshooting
def _print_accounts(accts, out=sys.stdout):
"""Print the list of accounts."""
count = len(accts)
- print(f'{count} account{"s" if count != 1 else ""} selected:\n', file=out)
+ print(f"{count} account{'s' if count != 1 else ''} selected:\n", file=out)
print(", ".join(str(a) for a in accts), file=out, end="\n\n")
@@ -1824,7 +1824,7 @@ Troubleshooting
_print_accounts(accts, out=sys.stderr)
print("Proceed (y/n)? ", flush=True, end="", file=sys.stderr)
answer = input()
- if not answer.lower() in ["y", "yes"]:
+ if answer.lower() not in ["y", "yes"]:
print("Exiting", file=sys.stderr)
sys.exit(0)
@@ -1922,6 +1922,37 @@ Functions
different names. The CLI uses the name of the shell script installed to
determine default path locations for commands as well as the default
session manager.
+
+
+Expand source code
+
+def main():
+ """The main entry point for the `*run` CLI tool installed with this package.
+
+ Runs the CLI tool. Exits with a `0` status code upon success. Upon error,
+ prints the error message to standard error. By default, a stack trace is not
+ included to minimize output. If the trace is desired, set the `AWSRUN_TRACE`
+ environment variable to `1`.
+
+ The CLI tool is installed on the system via the setup.py entry_points key.
+ There can be many instances of this command installed on the system with
+ different names. The CLI uses the name of the shell script installed to
+ determine default path locations for commands as well as the default
+ session manager.
+ """
+ try:
+ csp = _CSP.from_prog_name(sys.argv[0])
+ _cli(csp)
+
+ except Exception as e: # pylint: disable=broad-except
+ # Don't print stack traces by default as it can be overwhelming (scary)
+ # for those not familiar with Python development.
+ if os.getenv("AWSRUN_TRACE"):
+ traceback.print_exc(file=sys.stderr)
+
+ print(e, file=sys.stderr)
+ sys.exit(1)
+
@@ -2006,7 +2037,7 @@ Context
diff --git a/docs/cloudwatch.html b/docs/cloudwatch.html
index 4dbbe1d..934073f 100644
--- a/docs/cloudwatch.html
+++ b/docs/cloudwatch.html
@@ -3,13 +3,13 @@
-
+
awsrun.cloudwatch API documentation
-
+
@@ -453,6 +453,28 @@ Functions
ascending chronological order. The generator will yield a value for each
time interval even if CloudWatch has missing data points. In that case,
the value in the tuple will be a math.nan.
+
+
+Expand source code
+
+def get_metric(client, namespace, name, dimensions, statistic, last=3600, samples=60):
+ """Fetch data for the specified CloudWatch metric immediately.
+
+ This is a convenience method that should only be used if retrieving a
+ single metric. If multiple metrics are desired, it is more efficient to
+ use `CWMetric` instead.
+
+ Returns a generator object that yields a tuple containing a datetime
+ object and a corresponding value at that time. Values are returned in
+ ascending chronological order. The generator will yield a value for each
+ time interval even if CloudWatch has missing data points. In that case,
+ the value in the tuple will be a `math.nan`.
+ """
+ cwm = CWMetrics(client, last, samples)
+ get_values = cwm.add_metric(namespace, name, dimensions, statistic)
+ cwm.bulk_load()
+ return get_values()
+
@@ -745,6 +767,68 @@ Methods
will yield a value for each time interval even if CloudWatch has
missing data points. In that case, the value in the tuple will be a
math.nan.
+
+
+Expand source code
+
+def add_metric(self, namespace, name, dimensions, statistic):
+ """Queue the specified CloudWatch metric for bulk loading.
+
+ Use this method to register a metric for future retrieval via
+ `CWMetric.bulk_load`. Up to 500 metrics can be added. Once a metric
+ has been added, subsequent calls to `CWMetric.bulk_load` will retrieve
+ it again.
+
+ Returns a function that can be invoked after a bulk load has
+ completed. It will return a generator object that yields a tuple
+ containing a datetime object and a corresponding value at that time.
+ Values are returned in ascending chronological order. The generator
+ will yield a value for each time interval even if CloudWatch has
+ missing data points. In that case, the value in the tuple will be a
+ `math.nan`.
+ """
+ # We need to keep track of how many metrics are being retrieved as AWS
+ # only permits up to 500 in a single get_metric_data call. We also
+ # need a unique ID for each metric added as that is how we will match
+ # the response from the CW API.
+ self._counter += 1
+ if self._counter > _MAX_METRICS:
+ raise ValueError(f"number of metrics exceeded {_MAX_METRICS}")
+
+ # The AWS get_metric_data call requires a unique ID for each metric
+ # being retrieved. This ID is provided with the results, so the caller
+ # is able to match request with response. The ID only needs to be
+ # unique per call to get_metric_data, so we just use a simple counter
+ # as we'll never expose this ID to callers.
+ metric_id = f"id{self._counter}"
+
+ # Convert a dict in form of {"connId": "dxcon-aaa"} to the form
+ # required by AWS: [{"Name": "connId", "Value": "dxcon-aaa"}].
+ dimensions = [{"Name": n, "Value": v} for n, v in dimensions.items()]
+
+ # Create the query dict and append to the list of queries. The query
+ # is not executed until later when the user calls bulk_load().
+ self._queries.append(
+ {
+ "Id": metric_id,
+ "MetricStat": {
+ "Metric": {
+ "Namespace": namespace,
+ "MetricName": name,
+ "Dimensions": dimensions,
+ },
+ "Period": int(self._period),
+ "Stat": statistic,
+ },
+ "ReturnData": True,
+ }
+ )
+
+ # Return a closure to make it easy to retrieve results. After the user
+ # has called bulk_load, this function can be executed to obtain the
+ # results of the API call.
+ return lambda: self._get_metric_generator(metric_id)
+
def bulk_load(self)
@@ -758,6 +842,55 @@ Methods
Returns nothing. Results for each metric can be obtained by invoking
the callable returned from CWMetric.add_metric, which will return
the data associated with the last bulk load.
+
+
+Expand source code
+
+def bulk_load(self):
+ """Retrieve the metrics that have been queued.
+
+ This method will make a single API call to AWS to request all of the
+ requested metrics. This method may be called one or more times. Each
+ call will retrieve the metrics that were requested replacing the
+ results of a prior invocation.
+
+ Returns nothing. Results for each metric can be obtained by invoking
+ the callable returned from `CWMetric.add_metric`, which will return
+ the data associated with the last bulk load.
+ """
+ if not self._queries:
+ return
+
+ self._results = defaultdict(_MetricResult)
+ self._beg, self._end = self._compute_datetime_range()
+
+ for page in self._client.get_paginator("get_metric_data").paginate(
+ MetricDataQueries=self._queries,
+ StartTime=self._beg.isoformat(),
+ EndTime=self._end.isoformat(),
+ ScanBy="TimestampAscending",
+ ):
+ for mdr in page["MetricDataResults"]:
+ count = len(mdr["Values"])
+ _LOG.info(
+ "Results for %s (%s): count=%d status=%s",
+ mdr["Label"],
+ mdr["Id"],
+ count,
+ mdr["StatusCode"],
+ )
+ if count > 1:
+ _LOG.info(" [0]: %s %.2f", mdr["Timestamps"][0], mdr["Values"][0])
+ _LOG.info(" [1]: %s %.2f", mdr["Timestamps"][1], mdr["Values"][1])
+ _LOG.info("[-2]: %s %.2f", mdr["Timestamps"][-2], mdr["Values"][-2])
+ _LOG.info("[-1]: %s %.2f", mdr["Timestamps"][-1], mdr["Values"][-1])
+
+ # We use extend here because results can span multiple pages,
+ # so we just keep extending to the existing lists.
+ result = self._results[mdr["Id"]]
+ result.values.extend(mdr["Values"])
+ result.timestamps.extend(mdr["Timestamps"])
+
@@ -795,7 +928,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/cmdmgr.html b/docs/cmdmgr.html
index 8028e84..eb90889 100644
--- a/docs/cmdmgr.html
+++ b/docs/cmdmgr.html
@@ -3,13 +3,13 @@
-
+
awsrun.cmdmgr API documentation
-
+
@@ -596,6 +596,14 @@ Methods
-
Returns a dict of names and classes of all valid commands.
+Expand source code
+
+def commands(self):
+ """Returns a dict of names and classes of all valid commands."""
+ return self._loader.load_all()
+
def instantiate_command(self, command_name, argv, cfg)
@@ -614,6 +622,52 @@ Methods
CLI options defined by the command. See the documentation on
Config.get() for the parameters of the function.
If command_name is not found, raises CommandNotFoundError.
+
+
+Expand source code
+
+def instantiate_command(self, command_name, argv, cfg):
+ """Returns an instantiated command identified by `command_name`.
+
+ The `argv` parameter should be a a string of command line arguments
+ captured by `argparse.REMAINDER` in the main program that will be passed
+ directly to the command for processing via its static `from_cli` method.
+ If the arguments are not valid, the program will terminate by the arg
+ parser in the command. This is expected as the command's `argparse` will
+ present a user-friendly help message.
+
+ The `cfg` parameter is a function that can lookup key value pairs from a
+ user's configuration file. This is provided to the command author via
+ `Command.from_cli`, so default values can be provided for the any of the
+ CLI options defined by the command. See the documentation on
+ `awsrun.config.Config.get` for the parameters of the function.
+
+ If `command_name` is not found, raises `CommandNotFoundError`.
+ """
+ # Dynamically load the command specified by the user. All commands are
+ # really defined as modules in the 'commands' directory of this package
+ # by default, but this can be overridden by the --cmd-dir flag.
+ cmd_class = self._loader.load(command_name)
+
+ if not issubclass(cmd_class, Command):
+ raise TypeError(
+ f"'{command_name}' must be a subclass of awsrun.runner.Command"
+ )
+
+ # Create an argument parser for the command author, which is populated
+ # with the name and a help string from the command's module docstring.
+ parser = argparse.ArgumentParser(
+ command_name,
+ formatter_class=RawAndDefaultsFormatter,
+ epilog=sys.modules[cmd_class.__module__].__doc__,
+ )
+
+ # We then call the static method on the class to obtain an instance of
+ # the command. The command author is expected to parse whatever command
+ # line args the user passed on the command line. It is expected that the
+ # author terminate the program if incorrect arguments were passed.
+ return cmd_class.from_cli(parser, argv, cfg)
+
@@ -662,6 +716,17 @@ Methods
-
Returns the class object for the command called command_name.
If a valid class cannot be found, raises CommandNotFoundError.
+Expand source code
+
+def load(self, command_name):
+ """Returns the class object for the command called `command_name`.
+
+ If a valid class cannot be found, raises `CommandNotFoundError`.
+ """
+ raise NotImplementedError
+
def load_all(self)
@@ -670,6 +735,18 @@ Methods
Returns a dict of all valid commands found.
The keys of the dict are the command names and the values are the class
objects that have been loaded.
+Expand source code
+
+def load_all(self):
+ """Returns a dict of all valid commands found.
+
+ The keys of the dict are the command names and the values are the class
+ objects that have been loaded.
+ """
+ raise NotImplementedError
+
@@ -745,6 +822,26 @@ Methods
All loaders are searched for the command. If a command is found in
multiple loaders, the first loader containing the command is preferred.
If a valid class cannot be found, raises CommandNotFoundError.
+
+
+Expand source code
+
+def load(self, command_name):
+ """Returns the class object for the command called `command_name`.
+
+ All loaders are searched for the command. If a command is found in
+ multiple loaders, the first loader containing the command is preferred.
+ If a valid class cannot be found, raises `CommandNotFoundError`.
+ """
+ path_errors = {}
+ for loader in self.loaders:
+ try:
+ return loader.load(command_name)
+ except CommandNotFoundError as e:
+ path_errors.update(e.path_errors)
+
+ raise CommandNotFoundError(command_name, path_errors)
+
def load_all(self)
@@ -755,6 +852,26 @@ Methods
objects that have been loaded.
If a command is found in multiple
loaders, the first loader containing the command is preferred.
+
+
+Expand source code
+
+def load_all(self):
+ """Returns a dict of all valid commands found from all loaders.
+
+ The keys of the dict are the command names and the values are the class
+ objects that have been loaded. If a command is found in multiple
+ loaders, the first loader containing the command is preferred.
+ """
+ classes = {}
+ for loader in reversed(self.loaders):
+ # dict.update() replaces existing keys with new values, so reversing
+ # the list ensures that loaders at the beginning of the list take
+ # priority over those that come afterwards.
+ classes.update(loader.load_all())
+
+ return classes
+
@@ -995,7 +1112,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/access_report.html b/docs/commands/aws/access_report.html
index 2c0ec9d..8170f70 100644
--- a/docs/commands/aws/access_report.html
+++ b/docs/commands/aws/access_report.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.access_report API documentation
-
+
@@ -347,7 +347,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/aws.html b/docs/commands/aws/aws.html
index 6368faf..85dbd55 100644
--- a/docs/commands/aws/aws.html
+++ b/docs/commands/aws/aws.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.aws API documentation
-
+
@@ -1224,12 +1224,137 @@ Methods
-
Invoke an AWS CLI command for an account and region.
+Expand source code
+
+def regional_execute(self, session, acct, region):
+ """Invoke an AWS CLI command for an account and region."""
+
+ # We need to assemble a valid AWS cli command line that can be executed
+ # by the operating system. The instance variable awscli_args contains
+ # all arguments that follow 'aws': awsrun -r us-east-1 aws ... We will
+ # provide the --region argument as we know the region we are processing.
+ # We will also provide --output if the user has asked us to annotate an
+ # output type. This ensures we override any user settings that the AWS
+ # cli tool may pick up from ~/.aws/config.
+ cmd = [self.awscli_path, "--region", region]
+ if self.annotate:
+ cmd += ["--output", self.annotate]
+ elif self.output:
+ cmd += ["--output", self.output]
+ cmd += self.awscli_args
+ LOG.info("%s-%s: AWS CLI command: %s", acct, region, cmd)
+
+ # Before we can execute the AWS cli tool, we need to set a few env vars
+ # with our creds. Normally, the AWS cli expects this file to exist in
+ # the user's home directory at ~/.aws/credentials.
+ creds = session.get_credentials()
+
+ new_vars = {
+ "AWS_ACCESS_KEY_ID": creds.access_key,
+ "AWS_SECRET_ACCESS_KEY": creds.secret_key,
+ "AWS_SESSION_TOKEN": creds.token if creds.token else "",
+ }
+
+ env = ChainMap(new_vars, os.environ)
+
+ # We call run() and capture stdout and stderr from the command's
+ # output. Note: all the output is stored in memory, and then printed
+ # in collect_results. This means that if you run an AWS cli command
+ # that generates huge amounts of data, it'll all be stored in
+ # memory. Why don't we stream tho output from a pipe? We could use
+ # Popen directly, but if we returned from execute() before reading
+ # all of the results, then the worker will start another account, so
+ # in essence, all of the accounts will be "executed" immediately
+ # resulting in potentially many many AWS cli command processes
+ # running waiting for us to read the output.
+
+ result = subprocess.run(
+ cmd,
+ env=env,
+ check=False,
+ universal_newlines=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+
+ # Lastly, we return the ProcessCompleted object from the run() method.
+ # Recall, an awsrun command can return anything if you provide your own
+ # collect_results method.
+ return result
+
def regional_collect_results(self, acct, region, get_result)
-
Print the results to the console and files if specified.
+Expand source code
+
+def regional_collect_results(self, acct, region, get_result):
+ """Print the results to the console and files if specified."""
+
+ def annotate_lines(text, file=sys.stdout):
+ for line in filter(None, text.split("\n")):
+ print(f"{acct}/{region}: {line}", file=file, flush=True)
+
+ def annotate_json(text):
+ try:
+ d = {
+ "Account": str(acct),
+ "Region": region,
+ "Results": json.loads(text),
+ }
+ json.dump(d, sys.stdout, indent=4)
+ print()
+ except json.decoder.JSONDecodeError:
+ annotate_lines(
+ "Result of AWS CLI command is not valid JSON", file=sys.stderr
+ )
+
+ try:
+ # Let's get the return value from the execute method, which is the
+ # ProcessCompleted object from the subprocess.run() method above ...
+ result = get_result()
+
+ except Exception as e: # pylint: disable=broad-except
+ # ... unless there was an exception in which case it is raised by
+ # the call to get_result and we handle it here.
+ LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
+ annotate_lines(f"error: {e}", file=sys.stderr)
+ return
+
+ # Print stderr from AWS CLI always annotating the lines
+ annotate_lines(result.stderr, file=sys.stderr)
+
+ # Print stdout from AWS CLI annotating when appropriate
+ if not self.annotate:
+ print(result.stdout, end="", flush=True)
+ elif self.annotate == "json":
+ annotate_json(result.stdout)
+ elif self.annotate in ["text", "table"]:
+ annotate_lines(result.stdout)
+
+ # Save stdout and stderr from AWS CLI to disk if requested
+ if self.output_dir:
+ # Recall, the acct object passed to execute() can be anything. The
+ # str() method should provide us a unique means of identifying the
+ # account, but we need to escape any slashes if we use this as part
+ # of a filename so pathlib doesn't interpret as directories.
+ escaped = re.sub(r"[\\/]", "_", str(acct))
+ name = self.output_dir / f"{escaped}-{region}"
+
+ def save(suffix, text):
+ with name.with_suffix(suffix).open("w") as out:
+ out.write(text)
+
+ save(".stdout.log", result.stdout)
+ if result.stderr:
+ save(".stderr.log", result.stderr)
+
Inherited members
@@ -1292,7 +1417,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/cidr_overlap.html b/docs/commands/aws/cidr_overlap.html
index 499c74d..522df4c 100644
--- a/docs/commands/aws/cidr_overlap.html
+++ b/docs/commands/aws/cidr_overlap.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.cidr_overlap API documentation
-
+
@@ -384,7 +384,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/console.html b/docs/commands/aws/console.html
index a13fa78..98ed353 100644
--- a/docs/commands/aws/console.html
+++ b/docs/commands/aws/console.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.console API documentation
-
+
@@ -362,7 +362,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/dx_maint.html b/docs/commands/aws/dx_maint.html
index fa8ea44..c1ef87d 100644
--- a/docs/commands/aws/dx_maint.html
+++ b/docs/commands/aws/dx_maint.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.dx_maint API documentation
-
+
@@ -79,16 +79,17 @@ Module awsrun.commands.aws.dx_maint
Display Direct Connect maintenance events from AWS Health.
Overview
The dx_maint command displays a summary of Direct Connect maintenance events
-available from the AWS Health service sorted by the time AWS published the
-event, which is not necessarily the start of maintenance.
-Only recent events
-(past 7 days) and upcoming events are displayed by default.
+available from the AWS Health service sorted by the start time of the event.
+Only recent events (past 7 days) and upcoming events are displayed by default.
+The output includes the time the event was published (first date column), the
+start time of the event (second date column), and the end time of the event
+(last date column).
$ awsrun --account 111222333444 --account 222111444333 dx_maint --region us-east-1 --region us-west-2
111222333444 us-east-1 2021-11-19 09:11 CST closed MAINTENANCE_CANCELLED 2021-11-23 02:11 CST -> 2021-11-23 06:11 CST dxcon-aaaaaaaa available
111222333444 us-west-2 2021-12-07 05:12 CST closed MAINTENANCE_COMPLETE 2021-12-07 03:12 CST -> 2021-12-07 07:12 CST dxcon-bbbbbbbb available
111222333444 us-east-1 2022-01-18 02:01 CST closed MAINTENANCE_COMPLETE 2022-01-17 22:01 CST -> 2022-01-18 02:01 CST dxcon-cccccccc available
-222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
111222333444 us-east-1 2022-02-01 21:02 CST upcoming MAINTENANCE_SCHEDULED 2022-02-08 22:02 CST -> 2022-02-09 02:02 CST dxcon-eeeeeeee available
+222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
For non-emergency maintenance events that occurred in the past, the default
output only shows the COMPLETED or CANCELLED event to minimize on noise. For
@@ -156,16 +157,18 @@
Command Options
## Overview
The dx_maint command displays a summary of Direct Connect maintenance events
-available from the AWS Health service sorted by the time AWS published the
-event, which is not necessarily the start of maintenance. Only recent events
-(past 7 days) and upcoming events are displayed by default.
+available from the AWS Health service sorted by the start time of the event.
+Only recent events (past 7 days) and upcoming events are displayed by default.
+The output includes the time the event was published (first date column), the
+start time of the event (second date column), and the end time of the event
+(last date column).
$ awsrun --account 111222333444 --account 222111444333 dx_maint --region us-east-1 --region us-west-2
111222333444 us-east-1 2021-11-19 09:11 CST closed MAINTENANCE_CANCELLED 2021-11-23 02:11 CST -> 2021-11-23 06:11 CST dxcon-aaaaaaaa available
111222333444 us-west-2 2021-12-07 05:12 CST closed MAINTENANCE_COMPLETE 2021-12-07 03:12 CST -> 2021-12-07 07:12 CST dxcon-bbbbbbbb available
111222333444 us-east-1 2022-01-18 02:01 CST closed MAINTENANCE_COMPLETE 2022-01-17 22:01 CST -> 2022-01-18 02:01 CST dxcon-cccccccc available
- 222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
111222333444 us-east-1 2022-02-01 21:02 CST upcoming MAINTENANCE_SCHEDULED 2022-02-08 22:02 CST -> 2022-02-09 02:02 CST dxcon-eeeeeeee available
+ 222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
For non-emergency maintenance events that occurred in the past, the default
output only shows the COMPLETED or CANCELLED event to minimize on noise. For
@@ -416,12 +419,15 @@ Command Options
def post_hook(self):
# The awsrun post hook is called once after all accounts have been
# processed. At this point, we have collected everything we need in the
- # all_results instance variable. We sort this list by the last update
- # time as we want to display the maintenance events in the order they
- # were published by AWS. You might ask why don't we sort by the
- # maintenance start time? Because AWS sometimes does not publish a
- # start time for an event (I have no idea why).
- self.all_results.sort(key=lambda r: r[3]["lastUpdatedTime"])
+ # all_results instance variable. We sort this list by start time when
+ # possible, but the AWS API doesn't always send a startTime, so we
+ # fallback to the lastUpdatedTime, which seems to always be present.
+ self.all_results.sort(
+ key=lambda r: (
+ r[3].get("startTime", r[3]["lastUpdatedTime"]),
+ r[3]["lastUpdatedTime"],
+ )
+ )
for acct, region, dx, event in self.all_results:
for color, field in (
@@ -489,24 +495,64 @@ Functions
>>> list(chunk(l, 2))
[(1, 2), (3, 4), (5, 6), (7, 8), (9,)]
+
+
+Expand source code
+
+def chunk(it, n):
+ """Yield successive n-sized chunks from it.
+
+ >>> l = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+ >>> list(chunk(l, 3))
+ [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+ >>> list(chunk(l, 2))
+ [(1, 2), (3, 4), (5, 6), (7, 8), (9,)]
+ """
+ it = iter(it)
+ return iter(lambda: tuple(itertools.islice(it, n)), ())
+
def colorize(color, string)
-
Return a string wrapped with ASCII color.
+Expand source code
+
+def colorize(color, string):
+ """Return a string wrapped with ASCII color."""
+ return f"{color}{string}{Style.RESET_ALL}"
+
def date(date_time)
-
Return a human readable string for a datetime object.
+Expand source code
+
+def date(date_time):
+ """Return a human readable string for a datetime object."""
+ return date_time.strftime("%Y-%m-%d %H:%m %Z") if date_time else "not provided"
+
def shorten(event_type)
-
Trim 'AWS_DIRECTCONNECT_' from event_type name.
+Expand source code
+
+def shorten(event_type):
+ """Trim 'AWS_DIRECTCONNECT_' from event_type name."""
+ return re.sub("^AWS_DIRECTCONNECT_", "", event_type)
+
@@ -684,12 +730,15 @@ Classes
def post_hook(self):
# The awsrun post hook is called once after all accounts have been
# processed. At this point, we have collected everything we need in the
- # all_results instance variable. We sort this list by the last update
- # time as we want to display the maintenance events in the order they
- # were published by AWS. You might ask why don't we sort by the
- # maintenance start time? Because AWS sometimes does not publish a
- # start time for an event (I have no idea why).
- self.all_results.sort(key=lambda r: r[3]["lastUpdatedTime"])
+ # all_results instance variable. We sort this list by start time when
+ # possible, but the AWS API doesn't always send a startTime, so we
+ # fallback to the lastUpdatedTime, which seems to always be present.
+ self.all_results.sort(
+ key=lambda r: (
+ r[3].get("startTime", r[3]["lastUpdatedTime"]),
+ r[3]["lastUpdatedTime"],
+ )
+ )
for acct, region, dx, event in self.all_results:
for color, field in (
@@ -773,7 +822,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/dx_status.html b/docs/commands/aws/dx_status.html
index 83166cb..2cb58dd 100644
--- a/docs/commands/aws/dx_status.html
+++ b/docs/commands/aws/dx_status.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.dx_status API documentation
-
+
@@ -389,6 +389,7 @@ Command Options
"""
+
import io
import re
import sys
@@ -641,14 +642,14 @@ Command Options
Style.RESET_ALL,
Style.BRIGHT,
Fore.YELLOW,
- f"{c_name:22.22} ",
+ f"{c_name:25.25} ",
Style.RESET_ALL,
(Fore.GREEN if c_state == "AVAILABLE" else Fore.RED),
f"{c_state:10} ",
Fore.BLUE,
- f"{c_bandwidth:7} ",
+ f"{c_bandwidth:>9} ",
Fore.MAGENTA,
- f"{len(vifs):3} VIFs ",
+ f"{len(vifs):5} VIFs ",
(Fore.RED + f"({v_down} down)" if v_down > 0 else ""),
Fore.RESET,
]
@@ -722,6 +723,18 @@ Functions
-
Return 1 if n is 0, 0 if n is 1, otherwise n.
+Expand source code
+
+def invert(n):
+ """Return 1 if n is 0, 0 if n is 1, otherwise n."""
+ if n == 0:
+ return 1
+ if n == 1:
+ return 0
+ return n # NaN case
+
def bps(bandwidth)
@@ -731,6 +744,36 @@ Functions
The supported units include: Gbps, Mbps, Kbps, and bps (all case
insenstive). If the bandwidth string does not have a numeric component or
uses an unsuppored unit, a ValueError is raised.
+
+
+Expand source code
+
+def bps(bandwidth):
+ """Convert the bandwidth string to an integer in bits per second.
+
+ The supported units include: Gbps, Mbps, Kbps, and bps (all case
+ insenstive). If the bandwidth string does not have a numeric component or
+ uses an unsuppored unit, a ValueError is raised.
+ """
+ match = re.match(r"(\d+)\s*(\w+)", bandwidth)
+ if not match:
+ raise ValueError("bandwidth not in form of 10Mbps")
+
+ bw, units = match.groups()
+ bw = int(bw)
+ units = units.lower()
+
+ if units == "gbps":
+ return bw * 1000000000
+ if units == "mbps":
+ return bw * 1000000
+ if units == "kbps":
+ return bw * 1000
+ if units == "bps":
+ return bw
+
+ raise ValueError("unsupported unit, must be Gbps, Mbps, Kbps, Bps")
+
@@ -1001,7 +1044,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/index.html b/docs/commands/aws/index.html
index 95b300a..ed3fd9b 100644
--- a/docs/commands/aws/index.html
+++ b/docs/commands/aws/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws API documentation
-
+
@@ -217,7 +217,7 @@ Index
diff --git a/docs/commands/aws/kubectl.html b/docs/commands/aws/kubectl.html
index 5473509..c5b629e 100644
--- a/docs/commands/aws/kubectl.html
+++ b/docs/commands/aws/kubectl.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.kubectl API documentation
-
+
@@ -694,7 +694,8 @@ Module awsrun.commands.aws.kubectl
def _save_output(name, text):
- with name.open("w") as out:
+ fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w") as out:
out.write(text)
@@ -713,7 +714,7 @@ Module awsrun.commands.aws.kubectl
}
kubedir = Path.home() / Path(".kube")
- kubedir.mkdir(parents=True, exist_ok=True)
+ kubedir.mkdir(mode=0o700, parents=True, exist_ok=True)
filename = kubedir / Path(f"awsrun-{account_id}-{region}-{name}-{namespace}")
_save_output(filename, _KUBECONFIG.format(**substitions))
@@ -1027,6 +1028,76 @@ Methods
-
Print the results to the console and files if specified.
+Expand source code
+
+def regional_collect_results(self, acct, region, get_result):
+ """Print the results to the console and files if specified."""
+
+ def annotate_lines(result, text, file=sys.stderr):
+ prefix = f"{acct}/{region}"
+ if result.cluster and result.namespace:
+ prefix += f"/{result.cluster}/{result.namespace}"
+ for line in filter(None, text.split("\n") if text else ""):
+ print(f"{prefix}: {line}", file=file, flush=True)
+
+ def annotate_format(result, loader, dumper):
+ try:
+ d = {}
+ d["Account"] = str(acct)
+ d["Region"] = region
+ if result.cluster and result.namespace:
+ d["Cluster"] = result.cluster
+ d["Namespace"] = result.namespace
+ d["Results"] = loader(result.stdout)
+ dumper(d, sys.stdout, indent=4)
+ print()
+ except Exception as e: # pylint: disable=broad-except
+ annotate_lines(result, f"cannot parse output: {e}", file=sys.stderr)
+
+ try:
+ # Let's get the return value from the execute method, which is the
+ # ProcessCompleted object from the subprocess.run() method above ...
+ results = get_result()
+
+ except Exception as e: # pylint: disable=broad-except
+ # ... unless there was an exception in which case it is raised by
+ # the call to get_result and we handle it here.
+ LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
+ print(f"{acct}/{region}: error: {e}", file=sys.stderr)
+ return
+
+ for result in results:
+ # Print stderr from AWS CLI always annotating the lines
+ annotate_lines(result, result.stderr, file=sys.stderr)
+
+ # Print stdout from AWS CLI annotating when appropriate
+ if not self.annotate:
+ print(result.stdout, end="", flush=True)
+ elif self.annotate == "json":
+ annotate_format(result, json.loads, json.dump)
+ elif self.annotate == "yaml":
+ annotate_format(result, yaml.safe_load, yaml.dump)
+ elif self.annotate == "text":
+ annotate_lines(result, result.stdout)
+
+ # Save stdout and stderr from kubectl to disk if requested
+ if self.output_dir and result.cluster and result.namespace:
+ # Recall, the acct object passed to execute() can be anything. The
+ # str() method should provide us a unique means of identifying the
+ # account, but we need to escape any slashes if we use this as part
+ # of a filename so pathlib doesn't interpret as directories.
+ escaped = re.sub(r"[\\/]", "_", str(acct))
+ name = (
+ self.output_dir
+ / f"{escaped}-{region}-{result.cluster}-{result.namespace}"
+ )
+
+ _save_output(name.with_suffix(".stdout.log"), result.stdout)
+ if result.stderr:
+ _save_output(name.with_suffix(".stderr.log"), result.stderr)
+
Inherited members
@@ -1073,7 +1144,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/last.html b/docs/commands/aws/last.html
index 65feb5c..a88ef89 100644
--- a/docs/commands/aws/last.html
+++ b/docs/commands/aws/last.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.last API documentation
-
+
@@ -1639,30 +1639,72 @@ Methods
-
+
+
+Expand source code
+
+def all(self):
+ return self.events
+
def users(self)
-
+
+
+Expand source code
+
+def users(self):
+ return self.events_by_user.keys()
+
def by_user(self, user)
-
+
+
+Expand source code
+
+def by_user(self, user):
+ return self.events_by_user.get(user, [])
+
def by_id(self, event_id)
-
+
+
+Expand source code
+
+def by_id(self, event_id) -> Optional[_UserIdentityType]:
+ return self.events_by_key.get(event_id)
+
def filter(self, expr)
-
+
+
+Expand source code
+
+def filter(self, expr):
+ if not expr: # empty string?
+ events = self.unfiltered_events
+ else:
+ events = _filter_events(self.unfiltered_events, expr)
+
+ self.filter_expr = expr
+ self._load_events(events)
+ return len(self.events)
+
@@ -1768,7 +1810,7 @@ Class variables
var DEFAULT_CSS-
-
+
The type of the None singleton.
var Changed-
@@ -1780,11 +1822,11 @@
Class variables
var can_focus-
-
+
The type of the None singleton.
var can_focus_children-
-
+
The type of the None singleton.
Methods
@@ -1803,18 +1845,44 @@ Example
)
yield Footer()
+
+
+Expand source code
+
+def compose(self) -> ComposeResult:
+ yield Static(Markdown(self.help_md))
+ with Horizontal():
+ yield Input(placeholder=self.prompt)
+ yield Button("Close", variant="primary")
+
def on_input_submitted(self, event)
-
+
+
+Expand source code
+
+def on_input_submitted(self, event: Input.Submitted) -> None:
+ event.stop()
+ self.post_message(self.Changed(event.value))
+
-
+
+
+Expand source code
+
+def on_button_pressed(self, event: Button.Pressed) -> None:
+ event.stop()
+ self.post_message(self.Closed())
+
@@ -1861,23 +1929,15 @@ Ancestors
- textual.dom.DOMNode
- textual.message_pump.MessagePump
-Class variables
-
-var can_focus-
-
-
-var can_focus_children-
-
-
-
Inherited members
Popup:
Ancestors
- textual.dom.DOMNode
- textual.message_pump.MessagePump
-Class variables
-
-var can_focus-
-
-
-var can_focus_children-
-
-
-
Inherited members
Popup:
Class variables
var DEFAULT_CSS-
-
+
The type of the None singleton.
var SelectionChanged-
@@ -2054,11 +2106,11 @@
Class variables
var can_focus-
-
+
The type of the None singleton.
var can_focus_children-
-
+
The type of the None singleton.
Instance variables
@@ -2108,6 +2160,17 @@ Methods
+
+
+Expand source code
+
+def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
+ event.stop()
+ event.control.border_subtitle = (
+ f"{event.cursor_row + 1} of {len(event.control.rows)}"
+ )
+ self.post_message(self.SelectionChanged(event.row_key))
+
def compose(self)
@@ -2123,6 +2186,13 @@ Example
)
yield Footer()
+
+
+Expand source code
+
+def compose(self) -> ComposeResult:
+ yield DataTable()
+
def focus(self)
@@ -2136,12 +2206,28 @@ Args
Returns
The Widget instance.
+
+
+Expand source code
+
+def focus(self):
+ self.query_one(DataTable).focus()
+
def on_mount(self)
+
+
+Expand source code
+
+def on_mount(self):
+ dt = self.query_one(DataTable)
+ dt.cursor_type = "row"
+ dt.add_columns(*self.col_names)
+
@@ -2203,21 +2289,6 @@ Ancestors
textual.dom.DOMNode textual.message_pump.MessagePump Class variables
-
-var DEFAULT_CSS-
-
-
-var can_focus-
-
-
-var can_focus_children-
-
-
-
Methods
@@ -2225,19 +2296,43 @@ Methods
-
+
+
+Expand source code
+
+def on_mount(self):
+ dt = self.query_one(DataTable)
+ dt.show_header = False
+ dt.border_title = "Users"
+
def watch_contents(self, users)
-
+
+
+Expand source code
+
+def watch_contents(self, users):
+ dt = self.query_one(DataTable)
+ dt.clear()
+ for user in users:
+ dt.add_row(user, key=user)
+ dt.scroll_home()
+ dt.border_subtitle = f"0 of {len(users)}"
+
Inherited members
RowTable:
+DEFAULT_CSSSelectionChangedcan_focuscan_focus_childrencomposecontentsfocusAncestors
- textual.dom.DOMNode
- textual.message_pump.MessagePump
-Class variables
-
-var DEFAULT_CSS-
-
-
-var can_focus-
-
-
-var can_focus_children-
-
-
-
Methods
@@ -2327,19 +2407,45 @@ Methods
-
+
+
+Expand source code
+
+def on_mount(self):
+ self.query_one(DataTable).border_title = "Events"
+
def watch_contents(self, events)
-
+
+
+Expand source code
+
+def watch_contents(self, events):
+ dt = self.query_one(DataTable)
+ dt.clear()
+ for event in events:
+ row = event.to_row()
+ if event.has_error():
+ error = dt.app.get_css_variables()["error"]
+ row = [f"[{error}]{c}[/]" for c in row]
+ dt.add_row(*row, key=event.event_id())
+ dt.scroll_home()
+ dt.border_subtitle = f"0 of {len(events)}"
+
Inherited members
RowTable:
+DEFAULT_CSSSelectionChangedcan_focuscan_focus_childrencomposecontentsfocusClass variables
var CSS-
-
+
The type of the None singleton.
var TITLE-
-
+
The type of the None singleton.
var BINDINGS-
-
+
The type of the None singleton.
var theme-
-
+
The type of the None singleton.
Instance variables
@@ -2670,120 +2776,334 @@ Methods
-
Yield child widgets for a container.
This method should be implemented in a subclass.
+Expand source code
+
+def compose(self) -> ComposeResult:
+ yield FilterPopup() # Hidden off-screen initially
+ yield ExportPopup() # Hidden off-screen initially
+ yield Header()
+ yield Horizontal(
+ Vertical(UserTable(), EventTable(), id="left"),
+ EventDetail(),
+ id="main",
+ )
+ yield Footer()
+
def on_mount(self)
-
+
+
+Expand source code
+
+def on_mount(self):
+ self.query_one(Header).tall = True
+ self.query_one(FilterPopup).disabled = True
+ self.query_one(ExportPopup).disabled = True
+ self.total_count = self.filtered_count = len(self.events.all())
+ self.populate_data_tables()
+
def populate_data_tables(self, focus=True)
-
+
+
+Expand source code
+
+def populate_data_tables(self, focus=True):
+ ut = self.query_one(UserTable)
+ et = self.query_one(EventTable)
+ ed = self.query_one(EventDetail)
+
+ if not self.events.all():
+ ut.contents = et.contents = []
+ ed.event = None
+
+ elif not ut.has_class("hidden"):
+ ut.contents = sorted(self.events.users(), key=str.lower)
+ if focus:
+ ut.focus()
+
+ else:
+ et.contents = self.events.all()
+ if focus:
+ et.focus()
+
def action_toggle_dark(self)
-
An action to toggle dark mode.
+Expand source code
+
+def action_toggle_dark(self):
+ self.dark = not self.dark
+
+ # In EventDetail we define "event" as reactive with always_update,
+ # so this will force watch_event to be called which redraws the JSON
+ # with the correct theme. We need to do this as the content of the
+ # Static widget with our JSON is colored by rich which does not know
+ # about dark mode. So when user toggles, we need to syntax highlight
+ # the JSON with an appropriate theme.
+ ed = self.query_one(EventDetail)
+ ed.event = ed.event
+
def action_toggle_layout(self)
-
+
+
+Expand source code
+
+def action_toggle_layout(self):
+ self.toggle_class("vertical")
+
def action_copy(self)
-
+
+
+Expand source code
+
+def action_copy(self):
+ ed = self.query_one(EventDetail)
+ if ed.event:
+ pyperclip.copy(ed.event.to_json())
+
def action_filter_popup(self)
-
+
+
+Expand source code
+
+def action_filter_popup(self):
+ self.show_popup(FilterPopup)
+
def action_export_popup(self)
-
+
+
+Expand source code
+
+def action_export_popup(self):
+ self.show_popup(ExportPopup)
+
def action_toggle_users(self)
-
+
+
+Expand source code
+
+def action_toggle_users(self):
+ ut = self.query_one(UserTable)
+ ut.toggle_class("hidden")
+ self.populate_data_tables()
+
def dismiss_popup(self, name)
-
+
+
+Expand source code
+
+def dismiss_popup(self, name):
+ p = self.query_one(name)
+ p.disabled = True
+ p.add_class("offscreen")
+ self.set_focus(self.original_focus)
+ self.refresh(repaint=True, layout=True)
+
def show_popup(self, name)
-
+
+
+Expand source code
+
+def show_popup(self, name):
+ self.original_focus = self.query_one("*:focus")
+ p = self.query_one(name)
+ p.disabled = False
+ p.remove_class("offscreen")
+ p.query_one(Input).focus()
+
def on_filter_popup_closed(self, _)
-
+
+
+Expand source code
+
+def on_filter_popup_closed(self, _) -> None:
+ self.dismiss_popup(FilterPopup)
+
def on_filter_popup_changed(self, event)
-
+
+
+Expand source code
+
+def on_filter_popup_changed(self, event: FilterPopup.Changed) -> None:
+ # Only update the UI if the expression is different
+ if event.value != self.events.filter_expr:
+ self.filtered_count = self.events.filter(event.value)
+ self.query_one(EventDetail).filter_expr = event.value
+ self.populate_data_tables(focus=False)
+ self.dismiss_popup(FilterPopup)
+
def on_export_popup_closed(self, _)
-
+
+
+Expand source code
+
+def on_export_popup_closed(self, _) -> None:
+ self.dismiss_popup(ExportPopup)
+
def on_export_popup_changed(self, event)
-
+
+
+Expand source code
+
+def on_export_popup_changed(self, event: ExportPopup.Changed) -> None:
+ try:
+ with open(event.value, "w") as out:
+ json.dump(
+ [e.event for e in self.events.all()],
+ default=str,
+ indent=4,
+ fp=out,
+ )
+ self.dismiss_popup(ExportPopup)
+ except Exception:
+ self.bell()
+
def on_user_table_selection_changed(self, message)
-
+
+
+Expand source code
+
+def on_user_table_selection_changed(self, message: UserTable.SelectionChanged):
+ et = self.query_one(EventTable)
+ user = message.row_key
+ if user:
+ et.contents = self.events.by_user(user)
+
def on_event_table_selection_changed(self, message)
-
+
+
+Expand source code
+
+def on_event_table_selection_changed(self, message: EventTable.SelectionChanged):
+ ed = self.query_one(EventDetail)
+ event_key = message.row_key
+ if event_key:
+ event = self.events.by_id(event_key)
+ ed.event = event
+
def watch_total_count(self)
-
+
+
+Expand source code
+
+def watch_total_count(self):
+ self.update_sub_title()
+
def watch_filtered_count(self)
-
+
+
+Expand source code
+
+def watch_filtered_count(self):
+ self.update_sub_title()
+
def update_sub_title(self)
-
+
+
+Expand source code
+
+def update_sub_title(self):
+ s = f"{self.total_count} events loaded"
+ if self.filtered_count != self.total_count:
+ s += f", {self.filtered_count} matched"
+ self.sub_title = s
+
@@ -2880,15 +3200,15 @@ Class variables
var DEFAULT_CSS-
-
+
The type of the None singleton.
var can_focus-
-
+
The type of the None singleton.
var can_focus_children-
-
+
The type of the None singleton.
Instance variables
@@ -2938,6 +3258,13 @@ Methods
+
+
+Expand source code
+
+def on_mount(self):
+ self.border_title = "Event Detail"
+
def compose(self)
@@ -2953,12 +3280,37 @@ Example
)
yield Footer()
+
+
+Expand source code
+
+def compose(self) -> ComposeResult:
+ yield Static()
+
def watch_event(self, event)
+
+
+Expand source code
+
+def watch_event(self, event):
+ content = event.to_json() if event else ""
+ matching_lines = self._find_matching_lines(content)
+
+ self.query_one(Static).update(
+ Syntax(
+ content,
+ "json",
+ theme=EventViewer.theme["dark" if self.app.dark else "light"],
+ highlight_lines=matching_lines,
+ line_numbers=True,
+ )
+ )
+
@@ -3014,17 +3366,9 @@ FilterPopup
-
ExportPopup
-
RowTable
@@ -3045,9 +3389,6 @@
on_mountwatch_contentsDEFAULT_CSScan_focuscan_focus_children
@@ -3055,9 +3396,6 @@
on_mountwatch_contentsDEFAULT_CSScan_focuscan_focus_children
@@ -3109,7 +3447,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_hosted_zones.html b/docs/commands/aws/list_hosted_zones.html
index 7363b05..5e4abed 100644
--- a/docs/commands/aws/list_hosted_zones.html
+++ b/docs/commands/aws/list_hosted_zones.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_hosted_zones API documentation
-
+
@@ -273,7 +273,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_iam_policies.html b/docs/commands/aws/list_iam_policies.html
index 29b596d..b2bd7ed 100644
--- a/docs/commands/aws/list_iam_policies.html
+++ b/docs/commands/aws/list_iam_policies.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_iam_policies API documentation
-
+
@@ -919,6 +919,30 @@ Functions
account wildcards specified in the policy document. Returns true if the
document contains any of the actions in search_actions. Note: this does not
take into account whether an action is allowed or denied.
+
+
+Expand source code
+
+def has_actions(policy_doc, search_actions):
+ """Search a policy document, specifically the Action and NotAction blocks
+ for any IAM actions that match the search_actions. Matching does take into
+ account wildcards specified in the policy document. Returns true if the
+ document contains any of the actions in search_actions. Note: this does not
+ take into account whether an action is allowed or denied."""
+
+ for statement in make_list(policy_doc["Statement"]):
+ if "Action" in statement:
+ action_block = statement["Action"]
+ else:
+ action_block = statement["NotAction"]
+
+ for action in make_list(action_block):
+ pattern = re.compile("^" + re.escape(action).replace("\\*", ".*") + "$")
+ for search_action in search_actions:
+ if pattern.search(search_action):
+ return True
+ return False
+
def make_list(obj)
@@ -926,6 +950,19 @@ Functions
Returns obj if it is a list, otherwise returns a list of one element
containing obj. This is due to AWS's inconsistent use of JSON arrays.
+Expand source code
+
+def make_list(obj):
+ """Returns obj if it is a list, otherwise returns a list of one element
+ containing obj. This is due to AWS's inconsistent use of JSON arrays."""
+
+ if isinstance(obj, list):
+ return obj
+
+ return [obj]
+
def get_identities(collection, subresource, search_names)
@@ -935,6 +972,26 @@ Functions
identities from collection are returned. If search_names contains a list of
names, a resource is created by calling subresource. The resource is then
loaded to ensure it exists. All valid resources are returned.
+
+
+Expand source code
+
+def get_identities(collection, subresource, search_names):
+ """Return an iterable of IAM identities. If search_names is empty, then all
+ identities from collection are returned. If search_names contains a list of
+ names, a resource is created by calling subresource. The resource is then
+ loaded to ensure it exists. All valid resources are returned."""
+
+ if not search_names:
+ return collection.all()
+
+ # For each name being searched, create a resource object
+ identities = [subresource(name) for name in search_names]
+
+ # Resource objects load lazily, so we don't know if the identities
+ # above are valid or not. Let's filter out only the valid ones.
+ return filter(identity_exists, identities)
+
def identity_exists(identity)
@@ -942,6 +999,22 @@ Functions
Returns True if the identity exists, otherwise false. As a side
effect, the identity's resources are loaded.
+Expand source code
+
+def identity_exists(identity):
+ """Returns True if the identity exists, otherwise false. As a side
+ effect, the identity's resources are loaded."""
+
+ try:
+ identity.load()
+ return True
+ except ClientError as e:
+ if e.response["Error"]["Code"] == "NoSuchEntity":
+ return False
+ raise e
+
@@ -1227,12 +1300,59 @@ Methods
Prints the inline policies associated with identity.
+Expand source code
+
+def show_inline_policies(self, identity, ip):
+ """Prints the inline policies associated with identity."""
+ if not self.include_inline:
+ return
+
+ for inline in identity.policies.all():
+ # pylint: disable=cell-var-from-loop
+ # We wrap the policy_document in a lambda so boto3 resource is not
+ # fetched unless it is really needed. Although pylint complains
+ # about wrapping the looping var in a lambda, we use the lambda
+ # immediately if needed.
+ if self.should_skip(inline.policy_name, lambda: inline.policy_document):
+ continue
+
+ ip.print(f"policy=inline:{inline.policy_name}")
+ if self.verbose:
+ ip.print(json.dumps(inline.policy_document, indent=4), prefix=False)
+
def show_attached_policies(self, identity, ip)
Prints the attached policies associated with identity.
+Expand source code
+
+def show_attached_policies(self, identity, ip):
+ """Prints the attached policies associated with identity."""
+ if not self.include_attached:
+ return
+
+ for attached in identity.attached_policies.all():
+ # pylint: disable=cell-var-from-loop
+ # We wrap the default_version.document in a lambda so boto3 resource
+ # is not fetched unless it is really needed.
+ if self.should_skip(
+ attached.policy_name, lambda: attached.default_version.document
+ ):
+ continue
+
+ ip.print(f"policy=attached:{attached.policy_name}")
+ if self.verbose:
+ ip.print(
+ json.dumps(attached.default_version.document, indent=4),
+ prefix=False,
+ )
+
def should_skip(self, name, get_doc)
@@ -1242,6 +1362,41 @@ Methods
skipped.
For efficiency, the get_doc argument should be a function
that returns the policy document, so it is only called if needed.
+
+
+Expand source code
+
+def should_skip(self, name, get_doc):
+ """Returns false if the policy with name and policy document should be
+ skipped. For efficiency, the get_doc argument should be a function
+ that returns the policy document, so it is only called if needed."""
+
+ if self.search_policies and not any(
+ name.startswith(n) for n in self.search_policies
+ ):
+ return True
+ if self.not_search_policies and any(
+ name.startswith(n) for n in self.not_search_policies
+ ):
+ return True
+
+ # Short-circuit us out of here if we don't need to search for actions,
+ # which would require downloading the policy document. Recall, boto
+ # loads these things lazily, so if we don't need to access it, then
+ # don't load it.
+ if not self.search_actions and not self.not_search_actions:
+ return False
+
+ # Since we now need to search through the actual policy for action
+ # statements, invoke the function passed to actually get the policy.
+ doc = get_doc()
+ if self.search_actions and not has_actions(doc, self.search_actions):
+ return True
+ if self.not_search_actions and has_actions(doc, self.not_search_actions):
+ return True
+
+ return False
+
Inherited members
@@ -1289,6 +1444,17 @@ Methods
Print msg to buffer, if prefix is True, prepend the prefix.
+Expand source code
+
+def print(self, msg, prefix=True):
+ """Print msg to buffer, if prefix is True, prepend the prefix."""
+ if prefix:
+ print(f"{self.prefix} {msg}", file=self.out)
+ else:
+ print(msg, file=self.out)
+
@@ -1344,7 +1510,7 @@
diff --git a/docs/commands/aws/list_iam_roles.html b/docs/commands/aws/list_iam_roles.html
index e90046d..df94209 100644
--- a/docs/commands/aws/list_iam_roles.html
+++ b/docs/commands/aws/list_iam_roles.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_iam_roles API documentation
-
+
@@ -376,7 +376,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_igws.html b/docs/commands/aws/list_igws.html
index 26bcbae..895befd 100644
--- a/docs/commands/aws/list_igws.html
+++ b/docs/commands/aws/list_igws.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_igws API documentation
-
+
@@ -179,7 +179,7 @@ Command Options
file=out,
)
if attachments:
- print(f' vpcs={", ".join(attachments)}', end="", file=out)
+ print(f" vpcs={', '.join(attachments)}", end="", file=out)
print(file=out)
@@ -224,7 +224,7 @@ Classes
file=out,
)
if attachments:
- print(f' vpcs={", ".join(attachments)}', end="", file=out)
+ print(f" vpcs={', '.join(attachments)}", end="", file=out)
print(file=out)
@@ -285,7 +285,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_lambdas.html b/docs/commands/aws/list_lambdas.html
index 45d0f67..41d78fd 100644
--- a/docs/commands/aws/list_lambdas.html
+++ b/docs/commands/aws/list_lambdas.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_lambdas API documentation
-
+
@@ -213,7 +213,7 @@ Command Options
by_role[fn["Role"]].append(fn)
continue
print(
- f'{acct}/{region}: name={fn["FunctionName"]} runtime={fn["Runtime"]} role={fn["Role"]} public={_is_public(fn)}',
+ f"{acct}/{region}: name={fn['FunctionName']} runtime={fn['Runtime']} role={fn['Role']} public={_is_public(fn)}",
file=out,
)
@@ -287,7 +287,7 @@ Classes
by_role[fn["Role"]].append(fn)
continue
print(
- f'{acct}/{region}: name={fn["FunctionName"]} runtime={fn["Runtime"]} role={fn["Role"]} public={_is_public(fn)}',
+ f"{acct}/{region}: name={fn['FunctionName']} runtime={fn['Runtime']} role={fn['Role']} public={_is_public(fn)}",
file=out,
)
@@ -357,7 +357,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_public_ips.html b/docs/commands/aws/list_public_ips.html
index 1411357..8ccfbf8 100644
--- a/docs/commands/aws/list_public_ips.html
+++ b/docs/commands/aws/list_public_ips.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_public_ips API documentation
-
+
@@ -187,7 +187,7 @@ Command Options
# are processing.
for (vpc_id, owner_id), ips in public_ips.items():
print(
- f'{acct}/{region}: id={vpc_id} owner={owner_id} ips={", ".join(ips)}',
+ f"{acct}/{region}: id={vpc_id} owner={owner_id} ips={', '.join(ips)}",
file=out,
)
@@ -242,7 +242,7 @@ Classes
# are processing.
for (vpc_id, owner_id), ips in public_ips.items():
print(
- f'{acct}/{region}: id={vpc_id} owner={owner_id} ips={", ".join(ips)}',
+ f"{acct}/{region}: id={vpc_id} owner={owner_id} ips={', '.join(ips)}",
file=out,
)
@@ -303,7 +303,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_vpc_attribute.html b/docs/commands/aws/list_vpc_attribute.html
index 2322bbb..0099899 100644
--- a/docs/commands/aws/list_vpc_attribute.html
+++ b/docs/commands/aws/list_vpc_attribute.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_vpc_attribute API documentation
-
+
@@ -350,7 +350,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_vpcs.html b/docs/commands/aws/list_vpcs.html
index 983e9a6..22e5010 100644
--- a/docs/commands/aws/list_vpcs.html
+++ b/docs/commands/aws/list_vpcs.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_vpcs API documentation
-
+
@@ -264,7 +264,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/az.html b/docs/commands/azure/az.html
index 1971ea2..e5c9ea9 100644
--- a/docs/commands/azure/az.html
+++ b/docs/commands/azure/az.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.az API documentation
-
+
@@ -958,12 +958,151 @@ Methods
Invoke an Azure CLI command for an account.
+Expand source code
+
+def execute(self, session, acct):
+ """Invoke an Azure CLI command for an account."""
+
+ # We need to assemble a valid Azure CLI command line that can be
+ # executed by the operating system. The instance variable azureCLI_args
+ # contains all arguments that follow 'az': azurerun az ... We will
+ # provide --output if the user has asked us to annotate an output type.
+ # This ensures we override any user settings that the Azure CLI tool may
+ # pick up from ~/.azure directory.
+ cmd = [self.azurecli_path]
+ cmd += self.azurecli_args
+ cmd += ["--subscription", str(acct)]
+ if self.annotate:
+ cmd += ["--output", self.annotate]
+ elif self.output:
+ cmd += ["--output", self.output]
+ LOG.info("%s: Azure CLI command: %s", acct, cmd)
+
+ # Although the execute method receives a valid credential in the
+ # `session` argument, I've not found a way to pass that to the az CLI
+ # command. It doesn't matter though as az CLI users will simply use `az
+ # login` before running this azurerun wrapper.
+
+ # We call run() and capture stdout and stderr from the command's output.
+ # Note: all the output is stored in memory, and then printed in
+ # collect_results. This means that if you run an az CLI command that
+ # generates huge amounts of data, it'll all be stored in memory. Why
+ # don't we stream tho output from a pipe? We could use Popen directly,
+ # but if we returned from execute() before reading all of the results,
+ # then the worker will start another account, so in essence, all of the
+ # accounts will be "executed" immediately resulting in potentially many
+ # many Azure CLI command processes running waiting for us to read the
+ # output.
+
+ result = subprocess.run(
+ cmd,
+ check=False,
+ universal_newlines=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+
+ # Lastly, we return the ProcessCompleted object from the run() method.
+ # Recall, an azurerun command can return anything if you provide your
+ # own collect_results method.
+ return result
+
def collect_results(self, acct, get_result)
Print the results to the console and files if specified.
+Expand source code
+
+def collect_results(self, acct, get_result):
+ """Print the results to the console and files if specified."""
+
+ def annotate_lines(text, delimiter=": ", file=sys.stdout, separator=False):
+ for line in filter(None, text.split("\n")):
+ print(f"{acct}{delimiter}{line}", file=file, flush=True)
+ if separator and not text == "\n":
+ print()
+
+ def annotate_json(text):
+ try:
+ d = {
+ "Subscription": str(acct),
+ "Results": json.loads(text),
+ }
+ json.dump(d, sys.stdout, indent=4)
+ print()
+ except json.decoder.JSONDecodeError:
+ annotate_lines(
+ "Result of Azure CLI command is not valid JSON", file=sys.stderr
+ )
+
+ def annotate_yaml(text):
+ try:
+ d = {
+ "Subscription": str(acct),
+ "Results": yaml.safe_load(text),
+ }
+ yaml.safe_dump(d, sys.stdout, indent=4)
+ print("...") # end of yaml document separator
+ except yaml.representer.RepresenterError:
+ annotate_lines(
+ "Result of Azure CLI command is not valid JSON", file=sys.stderr
+ )
+
+ try:
+ # Let's get the return value from the execute method, which is the
+ # ProcessCompleted object from the subprocess.run() method above ...
+ result = get_result()
+
+ except Exception as e: # pylint: disable=broad-except
+ # ... unless there was an exception in which case it is raised by
+ # the call to get_result and we handle it here.
+ LOG.info("%s: error: %s", acct, e, exc_info=True)
+ annotate_lines(f"error: {e}", file=sys.stderr)
+ return
+
+ # Print stderr from Azure CLI always annotating the lines
+ annotate_lines(result.stderr, file=sys.stderr)
+
+ # Print stdout from Azure CLI annotating when appropriate
+ if not self.annotate:
+ if result.stdout not in ["", "\n"]: # skip blank output
+ print(
+ result.stdout,
+ end="\n" if self.output == "table" else "",
+ flush=True,
+ )
+ elif self.annotate == "json":
+ annotate_json(result.stdout)
+ elif self.annotate == "yaml":
+ annotate_yaml(result.stdout)
+ elif self.annotate == "table":
+ annotate_lines(result.stdout, separator=True)
+ elif self.annotate == "tsv":
+ annotate_lines(result.stdout, delimiter="\t")
+
+ # Save stdout and stderr from Azure CLI to disk if requested
+ if self.output_dir:
+ # Recall, the acct object passed to execute() can be anything. The
+ # str() method should provide us a unique means of identifying the
+ # account, but we need to escape any slashes if we use this as part
+ # of a filename so pathlib doesn't interpret as directories.
+ escaped = re.sub(r"[\\/]", "_", str(acct))
+ name = self.output_dir / f"{escaped}"
+
+ def save(suffix, text):
+ with name.with_suffix(suffix).open("w") as out:
+ out.write(text)
+
+ save(".stdout.log", result.stdout)
+ if result.stderr:
+ save(".stderr.log", result.stderr)
+
Inherited members
@@ -1022,7 +1161,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/cidr_overlap.html b/docs/commands/azure/cidr_overlap.html
index dd8cbcc..8e6b341 100644
--- a/docs/commands/azure/cidr_overlap.html
+++ b/docs/commands/azure/cidr_overlap.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.cidr_overlap API documentation
-
+
@@ -371,7 +371,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/index.html b/docs/commands/azure/index.html
index e8dcbab..14f9c82 100644
--- a/docs/commands/azure/index.html
+++ b/docs/commands/azure/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure API documentation
-
+
@@ -157,7 +157,7 @@ Index
diff --git a/docs/commands/azure/list_udrs.html b/docs/commands/azure/list_udrs.html
index a7c7191..a9f274b 100644
--- a/docs/commands/azure/list_udrs.html
+++ b/docs/commands/azure/list_udrs.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.list_udrs API documentation
-
+
@@ -235,7 +235,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/list_vnets.html b/docs/commands/azure/list_vnets.html
index 4db6d5b..d74dc47 100644
--- a/docs/commands/azure/list_vnets.html
+++ b/docs/commands/azure/list_vnets.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.list_vnets API documentation
-
+
@@ -216,7 +216,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/index.html b/docs/commands/index.html
index a65881a..83f7283 100644
--- a/docs/commands/index.html
+++ b/docs/commands/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands API documentation
-
+
@@ -1603,7 +1603,7 @@ Index
diff --git a/docs/config.html b/docs/config.html
index f8b44d2..cfc77ce 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -3,13 +3,13 @@
-
+
awsrun.config API documentation
-
+
@@ -880,6 +880,77 @@ Methods
c.get('path', 'to', 'value', type=And(StrMatch(r'\d+'), StrMatch(r'[A-Z]')))
c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
+
+
+Expand source code
+
+def get(self, *keys, default=None, type=None, must_exist=False):
+ """Return the specified value from the `Config`.
+
+ Specify the value to read by providing the keys required to reach the
+ value in the configuration. If the value is not found at the specified
+ key path, `None` or the `default` value is returned unless the
+ `must_exist` flag is `True`, in which case a `ValueError` is raised.
+
+ Values can be optionally type-checked to ensure it matches the specified
+ type. If the `type` matches the value in the configuration, the value is
+ returned, otherwise a `TypeError` is raised. Types are specified by
+ passing a `Type` object. There are numerous type objects defined in this
+ module. For example:
+
+ c.get('path', 'to', 'value', type=Int)
+ c.get('path', 'to', 'value', type=Bool)
+ c.get('path', 'to', 'value', type=Float)
+ c.get('path', 'to', 'value', type=Str)
+ c.get('path', 'to', 'value', type=StrMatch(r'^\\d+-\\d+$'))
+ c.get('path', 'to', 'value', type=IP)
+ c.get('path', 'to', 'value', type=List(IP))
+ c.get('path', 'to', 'value', type=List(Str))
+ c.get('path', 'to', 'value', type=List(Dict(Int, Str)))
+ c.get('path', 'to', 'value', type=Dict(Str, Int))
+ c.get('path', 'to', 'value', type=Or(Int, Float))
+ c.get('path', 'to', 'value', type=And(StrMatch(r'\\d+'), StrMatch(r'[A-Z]')))
+ c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
+ """
+ # pylint: disable=redefined-builtin
+
+ # This one-liner will recursively follow a list of keys into a
+ # dictionary and return the value. If a key does not exist, return an
+ # empty dict.
+ try:
+ value = reduce(lambda a, p: a.get(p, {}), keys, self.conf)
+ except AttributeError as e:
+ raise ValueError(
+ f"Error in config: {'->'.join(keys[:-1])}: not a dictionary"
+ ) from e
+
+ # If value is {} that means the key doesn't exist. If the must_exist
+ # flag was passed, then we raise a descriptive ValueError, otherwise we
+ # set it to the default.
+ if value == {}:
+ if must_exist:
+ raise ValueError(f"Error in config: {'->'.join(keys)}: must be set")
+ value = default
+
+ # If no value has been set in the config and none has been provided as a
+ # default, then return None.
+ if value is None:
+ return value
+
+ # If no type has been specified, then return the value in the config or
+ # the default without doing any type checking.
+ if not type:
+ return value
+
+ # Only return the value if it type checks correctly.
+ if type.type_check(value):
+ return value
+
+ # Finally, all other cases indicate a type error.
+ raise TypeError(
+ f"Error in config: {'->'.join(keys)}: not a {type}: {repr(value)}"
+ )
+
@@ -988,6 +1059,14 @@ Methods
Returns true if obj is a type matching this Type.
+Expand source code
+
+def type_check(self, obj):
+ """Returns true if obj is a type matching this `Type`."""
+ raise NotImplementedError
+
@@ -1636,7 +1715,7 @@ Dict
diff --git a/docs/index.html b/docs/index.html
index f0b96c0..33e4a2a 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -3,13 +3,13 @@
-
+
awsrun API documentation
-
+
@@ -316,7 +316,7 @@ Roadmap
"""
name = "awsrun"
-__version__ = "3.2.1"
+__version__ = "3.2.2"
@@ -418,7 +418,7 @@ Index
diff --git a/docs/plugins/accts/azure.html b/docs/plugins/accts/azure.html
index 5ab95ef..32dc13a 100644
--- a/docs/plugins/accts/azure.html
+++ b/docs/plugins/accts/azure.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.accts.azure API documentation
-
+
@@ -119,12 +119,12 @@ Module awsrun.plugins.accts.azure
: `AzureCLI` loads subscriptions and metadata for those subscriptions via the
Azure CLI `az account list --all` command.
"""
+
import logging
from awsrun.acctload import AzureCLIAccountLoader
from awsrun.plugmgr import Plugin
-
LOG = logging.getLogger(__name__)
@@ -435,7 +435,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/accts/index.html b/docs/plugins/accts/index.html
index 46dc4fa..854b341 100644
--- a/docs/plugins/accts/index.html
+++ b/docs/plugins/accts/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.accts API documentation
-
+
@@ -1010,7 +1010,6 @@ Module awsrun.plugins.accts
# Check and set auth options if using authentication.
if args.loader_auth != "none":
-
# Command line flags take priority
if args.loader_username:
auth_options["username"] = args.loader_username
@@ -2563,7 +2562,6 @@ Plug-in Options
# Check and set auth options if using authentication.
if args.loader_auth != "none":
-
# Command line flags take priority
if args.loader_username:
auth_options["username"] = args.loader_username
@@ -2680,7 +2678,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/creds/aws.html b/docs/plugins/creds/aws.html
index 85947a4..4a23f2f 100644
--- a/docs/plugins/creds/aws.html
+++ b/docs/plugins/creds/aws.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.creds.aws API documentation
-
+
@@ -1524,7 +1524,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/creds/azure.html b/docs/plugins/creds/azure.html
index b7db8e9..498a005 100644
--- a/docs/plugins/creds/azure.html
+++ b/docs/plugins/creds/azure.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.creds.azure API documentation
-
+
@@ -595,7 +595,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/creds/index.html b/docs/plugins/creds/index.html
index f7cc363..83515d8 100644
--- a/docs/plugins/creds/index.html
+++ b/docs/plugins/creds/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.creds API documentation
-
+
@@ -163,7 +163,7 @@ Index
diff --git a/docs/plugins/index.html b/docs/plugins/index.html
index cccfab8..c6d958d 100644
--- a/docs/plugins/index.html
+++ b/docs/plugins/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins API documentation
-
+
@@ -189,7 +189,7 @@ Index
diff --git a/docs/plugmgr.html b/docs/plugmgr.html
index 0aec0bb..1551b5d 100644
--- a/docs/plugmgr.html
+++ b/docs/plugmgr.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugmgr API documentation
-
+
@@ -204,6 +204,7 @@ Overview
`PluginManager.parse_args` as it will be called by `PluginManager.instantiate`
if it was not already.
"""
+
import importlib
import logging
from contextlib import suppress
@@ -529,6 +530,45 @@ Functions
the module to return. For example, some.module.MyClass will return the
class object MyClass from the Python module some.module. If the object
cannot be loaded, ImportError is raised.
+
+
+Expand source code
+
+def load_dotted_object(dotted_name):
+ """Returns the Python object found at the `dotted_name`.
+
+ `dotted_name` should include both the Python module as well as the object in
+ the module to return. For example, `some.module.MyClass` will return the
+ class object `MyClass` from the Python module `some.module`. If the object
+ cannot be loaded, `ImportError` is raised.
+ """
+
+ def doit(mod_name, attributes=None):
+ attributes = [] if attributes is None else attributes
+
+ if not mod_name:
+ raise ImportError(f"cannot import '{dotted_name}'")
+
+ mod = None
+ with suppress(ModuleNotFoundError):
+ mod = importlib.import_module(mod_name)
+
+ if not mod:
+ mod_name, _, attr = mod_name.rpartition(".")
+ attributes.append(attr)
+ return doit(mod_name, attributes)
+
+ attributes.reverse()
+ obj = reduce(lambda a, p: getattr(a, p, {}), attributes, mod)
+ if not obj:
+ raise ImportError(
+ f"module '{mod_name}' does not contain '{'.'.join(attributes)}'"
+ )
+
+ return obj
+
+ return doit(dotted_name)
+
@@ -729,6 +769,34 @@ Methods
It is perfectly acceptable to terminate the main program from
within this method. Alternatively, one can raise an exception which will
also terminate the program.
+
+
+Expand source code
+
+def instantiate(self, args):
+ """Returns an object created with options and arguments defined by the plug-in.
+
+ The `PluginManager` will invoke this method after it has completed
+ parsing the command line arguments that the plug-in defined in the
+ constructor. The `args` argument is a populated `argparse.Namespace`
+ object that contains the values of any command line arguments provided
+ by the user on the CLI.
+
+ This method also has access to the `self.parser` and `self.cfg` objects
+ that were provided in the constructor. The `self.cfg` object is useful
+ for cases where one does not want to provide a CLI flag for an option
+ defined in the configuration file. In this case, when instantiating the
+ object, you can pull values for the configuration file.
+
+ The `self.parser` object is useful if one wishes to abort the
+ instantiation of the plug-in. `argparse.ArgumentParser.error()` can be
+ used to provide an error message to the CLI user and terminate the
+ program. It is perfectly acceptable to terminate the main program from
+ within this method. Alternatively, one can raise an exception which will
+ also terminate the program.
+ """
+ raise NotImplementedError
+
@@ -996,6 +1064,61 @@ Methods
If the Accounts key does not exist in the configuration, then the
Identity plug-in will be used instead.
+
+
+Expand source code
+
+def parse_args(self, *keys, default=None):
+ """Load the plug-in and parse command line arguments passed via the CLI.
+
+ This method does not return anything, nor does it instantiate the
+ plug-in. It only loads the plug-in class and performs command line
+ argument processing for the `Plugin`. It is provided to allow one to
+ separate the parsing of all line arguments from the instantiation of
+ plug-ins. If this method is not explicitly called by the user, then it
+ will be implicitly called when `PluginManager.instantiate` is called.
+ See the `awsrun.plugmgr` documentation for the rationale.
+
+ The `keys` varargs specifies the path to the plug-in specification
+ contained with the configuration. See the `PluginManager` documentation
+ for details on the plug-in specification. If the path does not exist,
+ then the value of `default` is used instead. This default value must be
+ a dotted string pointing to a subclass of `Plugin`. For example:
+
+ pm = PluginManager(config, parser, parsed_args, unparsed_argv)
+ pm.parse_args('Accounts', default='awsrun.plugins.accts.Identity')
+
+ If the `Accounts` key does not exist in the configuration, then the
+ `awsrun.plugins.accts.Identity` plug-in will be used instead.
+ """
+ path = self._config.get(*keys, "plugin") or default
+ LOG.info("loading plug-in: %s", path)
+
+ try:
+ plugin_class = load_dotted_object(path)
+
+ except ImportError as e:
+ raise ValueError(f"Error in config: {'->'.join(keys)}->plugin: {e}") from e
+
+ if not (isclass(plugin_class) and issubclass(plugin_class, Plugin)):
+ raise TypeError(
+ f"Error in config: {'->'.join(keys)}->plugin: '{path}' is not a {Plugin}"
+ )
+
+ # Create a new config callable that points directly to the options
+ # stored in the configuration. This will make it easy for plugin authors
+ # to query the config without having to specify the key path leading up
+ # to the options section of the config.
+ cfg = partial(self._config.get, *keys, "options")
+
+ plugin = plugin_class(self._parser, cfg)
+ self.args, self.remaining_argv = self._parser.parse_known_args(
+ self.remaining_argv, self.args
+ )
+ LOG.info("parsed args=%s remaining args=%s", self.args, self.remaining_argv)
+
+ self._plugins[keys] = plugin
+
def instantiate(self, *keys, default=None, must_be=None)
@@ -1025,6 +1148,51 @@ Methods
must_be is provided, the returned value from Plugin.instantiate() must
be an instance of the type specified, otherwise a TypeError is raised.
If must_be is not provided, the returned object can be of any type.
+
+
+Expand source code
+
+def instantiate(self, *keys, default=None, must_be=None):
+ """Returns the instantiated plug-in.
+
+ This method ultimately returns the value from `Plugin.instantiate`,
+ which is passed a reference to the `PluginManager.args` object, so the
+ plug-in can use any parsed command line arguments it had requested. It
+ is usual, but not necessary, to invoke `PluginManager.parse_args` for
+ each plug-in before calling this method. This allows all command line
+ processing, and more important errors, to be complete before the actual
+ instantiation of any plug-ins.
+
+ The `keys` varargs specifies the path to the plug-in specification
+ contained with the configuration. See the `PluginManager` documentation
+ for details on the plug-in specification. If the path does not exist,
+ then the value of `default` is used instead. This default value must be
+ a dotted string pointing to a subclass of `Plugin`. For example:
+
+ pm = PluginManager(config, parser, parsed_args, unparsed_argv)
+ acct_loader = pm.instantiate(
+ 'Accounts',
+ must_be=AccountLoader,
+ default='awsrun.plugins.accts.Identity')
+
+ If the `Accounts` key does not exist in the configuration, then the
+ `awsrun.plugins.accts.Identity` plug-in will be used instead. If
+ `must_be` is provided, the returned value from `Plugin.instantiate` must
+ be an instance of the type specified, otherwise a `TypeError` is raised.
+ If `must_be` is not provided, the returned object can be of any type.
+ """
+ if keys not in self._plugins:
+ self.parse_args(*keys, default)
+
+ instance = self._plugins[keys].instantiate(self.args)
+
+ if must_be and not isinstance(instance, must_be):
+ raise TypeError(
+ f"Error in config: {'->'.join(keys)}->plugin: plugin did not build a {must_be}"
+ )
+
+ return instance
+
@@ -1072,7 +1240,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/runner.html b/docs/runner.html
index 59cf319..c0eab5e 100644
--- a/docs/runner.html
+++ b/docs/runner.html
@@ -3,13 +3,13 @@
-
+
awsrun.runner API documentation
-
+
@@ -1367,6 +1367,34 @@ Functions
account ID as a string from the account object.
Accounts are processed concurrently using a worker pool. The default number
of workers is specified by the max_workers argument, which defaults to 10.
+
+
+Expand source code
+
+def execute_function(session_provider, accounts, func, key=lambda x: x, max_workers=10):
+ """Executes a function across one or more accounts concurrently.
+
+ This is a convenience function that instantiates an `AccountRunner`, wraps a
+ `func` in a `CommandFunctionAdapter`, runs the command across a list of
+ `accounts`, and then returns a tuple of dicts representing the results and
+ errors after all accounts have been processed. The returned dicts are keyed
+ by the account.
+
+ The list of `accounts` can be a simple list of strings of account IDs or it
+ can be a list of objects that represent accounts. Passing objects can be
+ useful as each object is passed to `func` when processing an account. When
+ using account objects, you must provide a `key` function that returns the
+ account ID as a string from the account object.
+
+ Accounts are processed concurrently using a worker pool. The default number
+ of workers is specified by the `max_workers` argument, which defaults to 10.
+ """
+ command = CommandFunctionAdapter(func)
+ AccountRunner(session_provider, max_workers=max_workers).run(
+ command, accounts, key=key
+ )
+ return (command.results, command.errors)
+
def regional_execute_function(session_provider, accounts, regions, func, key=<function <lambda>>, max_workers=10)
@@ -1385,6 +1413,36 @@ Functions
account ID as a string from the account object.
Accounts are processed concurrently using a worker pool. The default number
of workers is specified by the max_workers argument, which defaults to 10.
+
+
+Expand source code
+
+def regional_execute_function(
+ session_provider, accounts, regions, func, key=lambda x: x, max_workers=10
+):
+ """Executes a function across one or more accounts and regions concurrently.
+
+ This is a convenience function that instantiates an `AccountRunner`, wraps a
+ `func` in a `RegionalCommandFunctionAdapter`, runs the command across a list
+ of `accounts` and `regions`, and then returns a tuple of dicts representing
+ the results and errors after all accounts have been processed. The returned
+ dicts are keyed by a tuple of account and region.
+
+ The list of `accounts` can be a simple list of strings of account IDs or it
+ can be a list of objects that represent accounts. Passing objects can be
+ useful as each object is passed to `func` when processing an account. When
+ using account objects, you must provide a `key` function that returns the
+ account ID as a string from the account object.
+
+ Accounts are processed concurrently using a worker pool. The default number
+ of workers is specified by the `max_workers` argument, which defaults to 10.
+ """
+ command = RegionalCommandFunctionAdapter(regions, func)
+ AccountRunner(session_provider, max_workers=max_workers).run(
+ command, accounts, key=key
+ )
+ return (command.results, command.errors)
+
def max_thread_limit(count)
@@ -1407,6 +1465,45 @@ Functions
Note: while this decorator can limit the number of concurrent executions, it
will not increase the number of workers in the AccountRunner worker pool.
This is a rate limiting decorator only.
+
+
+Expand source code
+
+def max_thread_limit(count):
+ """Decorator to limit maximum number of concurrent executions.
+
+ In some cases, the author of a `Command` may wish to restrict the number of
+ concurrent executions of a command, regardless of the number of workers the
+ `AccountRunner` has been instantiated with. awsrun CLI users can specify the
+ number of workers via the `--threads` flag, which defaults to 10.
+
+ Use this with `Command.execute` or `RegionalCommand.regional_execute` to
+ guarantee concurrent executions do not exceed the specified `count`. When
+ the number of concurrent executions exceed the value, they will block until
+ an existing execution has completed. For example, the following will limit
+ concurrent executions to one:
+
+ @max_thread_limit(1)
+ def regional_execute(self, session, acct, region):
+ pass
+
+ Note: while this decorator can limit the number of concurrent executions, it
+ will not increase the number of workers in the `AccountRunner` worker pool.
+ This is a rate limiting decorator only.
+ """
+
+ def decorator(func):
+ sem = threading.BoundedSemaphore(count)
+
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ with sem:
+ return func(*args, **kwargs)
+
+ return wrapper
+
+ return decorator
+
def get_paginated_resources(client, paginator, page_key, predicate=<function <lambda>>, **kwargs)
@@ -1456,6 +1553,73 @@ Functions
lambda l: l["Protocol"] in ["UDP", "TCP_UDP"],
LoadBalancer=lb_arn)
+
+
+Expand source code
+
+def get_paginated_resources(
+ client,
+ paginator,
+ page_key,
+ predicate: Callable[[dict], bool] = lambda _: True,
+ **kwargs,
+):
+ """Return the full list of boto3 resources via a paginator.
+
+ `client` is a boto3 client. `paginator` is the name (string) of the
+ paginator. `page_key` is the name (string) of the dictionary key used in
+ the paginated responses.
+
+ `predicate` is a function that determines which resources are included in
+ the list. The function takes a single argument, the resource. If it returns
+ `True`, the resource is included, otherwise it is excluded.
+
+ The remaining `kwargs` are used to specify which resources to retrieve. Some
+ paginators do not require any additional arguments, but others do to
+ restrict the size of the response.
+
+ For example, to collect the list of load balancers, one might write:
+
+ def get_lbs(client):
+ lbs = []
+ paginator = client.get_paginator("describe_load_balancers")
+ for page in paginator.paginate():
+ for lb in page["LoadBalancers"]:
+ lbs.append(lb)
+ return lbs
+
+ This can be simplified by using `get_paginated_resources`:
+
+ lbs = get_paginated_resources(client, "describe_load_balancers", "LoadBalancers")
+
+ As another example, to collect the list of UDP listeners for a load
+ balancer, one might write:
+
+ def get_listeners(client, lb_arn):
+ listeners = []
+ paginator = client.get_paginator("describe_listeners")
+ for page in paginator.paginate(LoadBalancerArn=lb_arn):
+ for listener in page["Listeners"]:
+ if listener["Protocol"] in ["UDP", "TCP_UDP"]:
+ listeners.append(listener)
+ return listeners
+
+ This, too, can be simplified using `get_paginated_resources`:
+
+ listeners = get_paginated_resources(
+ client,
+ "describe_listeners",
+ "Listeners",
+ lambda l: l["Protocol"] in ["UDP", "TCP_UDP"],
+ LoadBalancer=lb_arn)
+ """
+ resources = []
+ for page in client.get_paginator(paginator).paginate(**kwargs):
+ for resource in page[page_key]:
+ if predicate(resource):
+ resources.append(resource)
+ return resources
+
@@ -1831,6 +1995,25 @@ Methods
rather once before any accounts are processed.
To provide backwards compatibility, the default implementation invokes
Command.pre_hook().
+
+
+Expand source code
+
+def pre_hook_with_context(self, context):
+ """Invoked by `AccountRunner.run` before any account processing starts.
+
+ This method is invoked only once per invocation of `AccountRunner.run`.
+ The `context` parameter is the opaque object passed as the context
+ parameter to `AccountRunner.run`. It is intended to provide access to
+ additional runtime context information for the command to leverage.
+ The method is not executed before each account is processed, but
+ rather once before any accounts are processed.
+
+ To provide backwards compatibility, the default implementation invokes
+ `Command.pre_hook`.
+ """
+ self.pre_hook()
+
def pre_hook(self)
@@ -1843,6 +2026,22 @@ Methods
executed before each account is processed, but rather once before
any accounts are processed.
The default implementation does nothing.
+
+
+Expand source code
+
+def pre_hook(self):
+ """Invoked in the default implementation of `pre_hook_with_context`
+ before any account processing starts.
+
+ This method is invoked in the event a command does not override the
+ default implementation of `pre_hook_with_context`. The method is not
+ executed before each account is processed, but rather once before
+ any accounts are processed.
+
+ The default implementation does nothing.
+ """
+
def post_hook(self)
@@ -1853,6 +2052,20 @@ Methods
It is not executed after each account is processed, but rather once
after all accounts have been processed.
The default implementation does nothing.
+
+
+Expand source code
+
+def post_hook(self):
+ """Invoked by `AccountRunner.run` after all processing has completed.
+
+ This method is invoked only once per invocation of `AccountRunner.run`.
+ It is not executed after each account is processed, but rather once
+ after all accounts have been processed.
+
+ The default implementation does nothing.
+ """
+
def execute(self, session, acct)
@@ -1901,6 +2114,54 @@ Methods
+
+
+Expand source code
+
+def execute(self, session, acct):
+ """Invoked by `AccountRunner.run` to process an account.
+
+ This method is invoked once for each account. The `session` parameter is
+ a boto3 Session object, or similar cloud SDK session-like object, with
+ credentials for the account being processed. The `acct` will be the same
+ object that was passed in the list of accounts to `AccountRunner.run`.
+
+ The return value from this method can be of any type. By default, this
+ value will be printed to the console if a `Command.collect_results`
+ implementation is not provided. If an exception is raised during the
+ execution, it will be printed to the console on standard error.
+
+ Note the following items of importance:
+
+ 1. Command authors must implement this method. There is no default
+ implementation.
+
+ 2. Recognize that this method will be invoked concurrently, so
+ modification of instance variables from within this method requires
+ synchronization. If accumulating results, define a custom
+ `Command.collect_results` which is guaranteed to be invoked
+ sequentially.
+
+ 3. Do not call `sys.exit` or the entire program will terminate. The
+ proper way to exit from this method is either by returning a value or
+ by raising an exception.
+
+ 4. Do not print directly to the console as output will be interspersed
+ with other output from other concurrently running threads processing
+ other accounts. The best practice when printing is to accumulate a
+ string buffer and return the buffer at the end of the method:
+
+ def execute(self, session, acct):
+ out = io.StringIO()
+
+ # Do stuff
+ print('This will be printed to the console eventually', file=out)
+ # Do more stuff
+
+ return out.getvalue()
+ """
+ raise NotImplementedError
+
def collect_results(self, acct, get_result)
@@ -1922,6 +2183,38 @@ Methods
to the console on standard output. If the command's execution raises an
exception, it prints the exception to standard error and logs the stack
trace at WARN log level.
+
+
+Expand source code
+
+def collect_results(self, acct, get_result):
+ """Invoked by `AccountRunner.run` after processing an account.
+
+ This method is invoked by `AccountRunner.run` after each `acct` has been
+ processed by `Command.execute`. The results from execute are provided
+ via `get_result`, which is a callable that will either return the value
+ returned by execute or raise an exception if one was raised. The `acct`
+ parameter will be the same object that was passed in the list of
+ accounts to `AccountRunner.run`.
+
+ Note: `Command.collect_results` is guaranteed to be called sequentially
+ by the main thread, so it is safe to mutate instance variables attached
+ to the `Command` object from within this method. This allows command
+ authors to safely accumulate results of processing within instance
+ variables without the need for synchronization.
+
+ The default implementation prints the return value of `Command.execute`
+ to the console on standard output. If the command's execution raises an
+ exception, it prints the exception to standard error and logs the stack
+ trace at WARN log level.
+ """
+ try:
+ print(get_result(), end="", flush=True)
+
+ except Exception as e: # pylint: disable=broad-except
+ LOG.warning("%s: error: %s", acct, e, exc_info=True)
+ print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
+
@@ -2287,6 +2580,65 @@ Methods
+
+
+Expand source code
+
+def regional_execute(self, session, acct, region):
+ """Invoked by `AccountRunner.run` to process an account / region pair.
+
+ This method is invoked for each account and region pair. The `session`
+ parameter is a boto3 Session object, or similar cloud SDK session-like
+ object, with credentials for the account being processed. The `acct`
+ will be the same object that was passed in the list of accounts to
+ `AccountRunner.run`. `region` is a string representing the region name
+ such as "us-east-1".
+
+ The return value can be of any type. It will be, by default, printed to
+ the console if a `RegionalCommand.regional_collect_results`
+ implementation is not provided. If an exception is raised during the
+ execution, it will be printed to the console on standard error.
+
+ Note the following items of importance:
+
+ 1. Command authors must implement this method. There is no default
+ implementation.
+
+ 2. Recognize that this method will be invoked concurrently, so
+ modification of instance variables from within this method requires
+ synchronization. If accumulating results, define a custom
+ `RegionalCommand.regional_collect_results` which is guaranteed to be
+ invoked sequentially.
+
+ 3. Although accounts are processed concurrently, the regions are
+ processed sequentially for each account. This ensures that multiple
+ regions for the same account are never executed at the same time. It
+ provides command authors a guarantee that an account and all its
+ regions will be processed sequentially. The same session object and
+ credentials are provided to this method for each region being
+ processed. Credentials could expire if processing of regions takes a
+ significant amount of time.
+
+ 4. Do not call `sys.exit` or the entire program will terminate. The
+ proper way to exit from this method is either by returning a value or
+ by raising an exception.
+
+ 5. Do not print directly to the console as output will be interspersed
+ with other output from other concurrently running threads processing
+ other accounts. The best practice when printing is to accumulate a
+ string buffer and return the buffer at the end of the method:
+
+ def regional_execute(self, session, acct, region):
+ out = io.StringIO()
+
+ # Do stuff
+ print('This will be printed to the console eventually', file=out)
+ # Do more stuff
+
+ return out.getvalue()
+ """
+ raise NotImplementedError
+
def regional_collect_results(self, acct, region, get_result)
@@ -2310,6 +2662,40 @@ Methods
RegionalCommand.regional_execute() to the console on standard output. If
the command's execution raises an exception, it prints the exception to
standard error and logs the stack trace at WARN log level.
+
+
+Expand source code
+
+def regional_collect_results(self, acct, region, get_result):
+ """Invoked by `AccountRunner.run` after processing an account and region.
+
+ This method is invoked by `AccountRunner.run` after each `acct` /
+ `region` pair has been processed by `RegionalCommand.regional_execute`.
+ The results from execute are provided via `get_result`, which is a
+ callable that will either return the value returned by the regional
+ execute method or raise an exception if one was raised. The `acct`
+ parameter will be the same object that was passed in the list of
+ accounts to `AccountRunner.run`.
+
+ Note: `RegionalCommand.regional_collect_results` is guaranteed to be
+ called sequentially by the main thread, so it is safe to mutate instance
+ variables attached to the `RegionalCommand` object from within this
+ method. This allows command authors to safely accumulate results of
+ processing within instance variables without the need for
+ synchronization.
+
+ The default implementation prints the return value of
+ `RegionalCommand.regional_execute` to the console on standard output. If
+ the command's execution raises an exception, it prints the exception to
+ standard error and logs the stack trace at WARN log level.
+ """
+ try:
+ print(get_result(), end="", flush=True)
+
+ except Exception as e: # pylint: disable=broad-except
+ LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
+ print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
+
Inherited members
@@ -2683,6 +3069,115 @@ Methods
for the caller to pass a runtime context to the command being
executed. See the Context With Pre-Hook
section of the use guide for an example.
+
+
+Expand source code
+
+def run(self, cmd, accounts, key=lambda x: x, context=None):
+ """Execute a command concurrently on the specified accounts.
+
+ This method will block until all accounts have been processed. The
+ return value is the number of seconds it took to process the accounts.
+
+ The `cmd` must be a subclass of `Command`. The runner will invoke
+ `Command.pre_hook_with_context` once before it starts processing any
+ accounts passing it the `context` argument, which is simply passed
+ through as an opaque object. This can be used to pass a runtime context
+ to a `Command`.
+
+ After the pre-hook has been invoked, accounts are then processed
+ concurrently and `Command.execute` is invoked by a worker for each
+ account. As each execute method returns, the main thread will invoke
+ `Command.collect_results`, which ensures results are collected
+ sequentially. Finally, after all accounts have been processed,
+ `Command.post_hook` is called.
+
+ The specified list of `accounts` can be of any type as long as the
+ function specified by the `key` parameter returns a string representing
+ the cloud account ID when passed one of these accounts. This allows
+ users to pass any object representing an account all the way through to
+ `Command.execute`. The only contract is that a `key` function must be
+ provided, so workers can obtain the account ID, which is used to request
+ a session for the account.
+
+ For example, `accounts` could be a simple list of strings of AWS account
+ IDs. The default value of `key` is the identity function, which returns
+ the string itself satisfying the contract above. Alternatively,
+ `accounts` could be a list of dicts containing metadata for an account,
+ which would then be available for command authors in `Command.execute`.
+ If the list of accounts specified contained the following:
+
+ [
+ {'id': '100200300400', 'env': 'prod', 'status': 'active'},
+ {'id': '200300400100', 'env': 'dev', 'status': 'active'},
+ {'id': '300400100200', 'env': 'dev', 'status': 'active'},
+ ]
+
+ Then, the `key` argument must be specified as `lambda x: x['id']`, which
+ will return the account ID string satisfying the contract above.
+ Likewise, if accounts were a list of objects that contained an `acct_id`
+ attribute, `key` must be defined as `lambda x: x.acct_id` to satisfy the
+ contract. If the key function does not return a string or throws an
+ exception, then an `InvalidAccountIDError` is raised in the worker
+ thread processing the account, which will then propagate to the
+ `Command.collect_results`.
+
+ The optional `context` parameter can be any value. It is passed
+ as-is to `Command.pre_hook_with_context`. It provides a mechanism
+ for the caller to pass a runtime context to the command being
+ executed. See the [Context With Pre-Hook](#context-with-pre-hook)
+ section of the use guide for an example.
+ """
+ # This will ensure v1 users of awsrun aren't mixing v1 Command's with
+ # the v2 framework.
+ if not isinstance(cmd, Command):
+ raise TypeError(f"'{cmd}' must be a subclass of awsrun.runner.Command")
+
+ # Wrapper to ensure the user-supplied key function returns an string of
+ # digits (an AWS account id). It will throw an InvalidAccountIDError
+ # if there are any exceptions thrown from use of their key function.
+ key = _valid_key_fn(key)
+
+ start = time.time()
+ cmd.pre_hook_with_context(context)
+
+ with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
+ # The worker task processes a single account. The worker task takes
+ # care to capture the result of the command's execute method. We
+ # don't want a poorly written command that raises an exception to
+ # terminate the main program, so the return value of the command's
+ # execute method or any exception raised is wrapped in a callable
+ # that is provided back to the command via its collect_results
+ # method. When the callable is later invoked, it will return the
+ # return value from execute or it will raise the caught exception.
+ def worker_task(acct):
+ try:
+ acct_id = key(acct) # Get the acct id from the account obj
+ session = self.session_provider.session(acct_id)
+ return _wrap_result(cmd.execute, session, acct)
+
+ except Exception as e: # pylint: disable=broad-except
+ # NOTE: exceptions thrown by a Command's execute are not
+ # handled in this block, but in wrap_result above. This
+ # block handles exceptions that occur while obtaining a
+ # session for the account.
+ return _wrap_exception(e)
+
+ # Submit all of the jobs for execution to the thread pool.
+ f2a = {pool.submit(worker_task, a): a for a in accounts}
+
+ # NOTE: collect_results is called by the main thread sequentially
+ # after each worker completes their task. This is a guarantee for
+ # Command authors as it allows them to safely update instance vars
+ # in the Command because it is not safe to do so in the execute
+ # method which is invoked in a concurrently running worker thread.
+ for future in as_completed(f2a):
+ acct = f2a[future]
+ cmd.collect_results(acct, future.result())
+
+ cmd.post_hook()
+ return time.time() - start
+
@@ -2793,7 +3288,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/session/aws.html b/docs/session/aws.html
index a9d5e1d..12046dc 100644
--- a/docs/session/aws.html
+++ b/docs/session/aws.html
@@ -3,13 +3,13 @@
-
+
awsrun.session.aws API documentation
-
+
@@ -951,6 +951,38 @@ Methods
event this method is invoked multiple times for the same account. Users
are guaranteed that the credentials will be valid for half of the cache
duration time specified in the constructor.
+
+
+Expand source code
+
+def session(self, acct_id):
+ """Returns a boto3 Session with credentials for the requested account.
+
+ The `acct_id` is a string containing the AWS account ID. The returned
+ boto3 Session object is ready to use and loaded with the requested
+ credentials.
+
+ The credentials loaded into the boto3 Session object are cached in the
+ event this method is invoked multiple times for the same account. Users
+ are guaranteed that the credentials will be valid for half of the cache
+ duration time specified in the constructor.
+ """
+ with self._lock:
+ # setdefault is technically atomic in cpython 2.7 or 3.2 higher, so
+ # the lock may seem redundant, but it's safer to make this explicit.
+ ev = self._creds.setdefault(
+ (acct_id, self._role),
+ ExpiringValue(lambda: self.credentials(acct_id), self._duration / 2),
+ )
+
+ creds = ev.value()
+
+ return boto3.Session(
+ aws_access_key_id=creds["AccessKeyId"],
+ aws_secret_access_key=creds["SecretAccessKey"],
+ aws_session_token=creds["SessionToken"],
+ )
+
def credentials(self, acct_id)
@@ -961,6 +993,21 @@ Methods
dict must include the following keys: "AccessKeyId", "SecretAccessKey",
and "SessionToken". This dict will be cached by the session provider.
Refer to the module documentation for the exceptions that may be raised.
+
+
+Expand source code
+
+def credentials(self, acct_id):
+ """Returns a dict containing AWS credentials for the requested account.
+
+ The `acct_id` is a string containing the AWS account ID. The returned
+ dict must include the following keys: "AccessKeyId", "SecretAccessKey",
+ and "SessionToken". This dict will be cached by the session provider.
+
+ Refer to the module documentation for the exceptions that may be raised.
+ """
+ raise NotImplementedError
+
@@ -1239,6 +1286,19 @@ Methods
refreshed first, then returned.
See the module documentation for the
exceptions that may be raised.
+
+
+Expand source code
+
+def assertion(self, refresh=False):
+ """Returns a SAML assertion from the IdP.
+
+ This value is cached by default. If refresh is True, the value is
+ refreshed first, then returned. See the module documentation for the
+ exceptions that may be raised.
+ """
+ return self._cached_saml.value(refresh)
+
Inherited members
@@ -1491,7 +1551,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/session/azure.html b/docs/session/azure.html
index fddd162..ea1561b 100644
--- a/docs/session/azure.html
+++ b/docs/session/azure.html
@@ -3,13 +3,13 @@
-
+
awsrun.session.azure API documentation
-
+
@@ -204,6 +204,7 @@ Thread Safety
token, to populate the token cache or refresh it, before allowing other threads
to proceed concurrently.
"""
+
import functools
import threading
@@ -540,7 +541,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/session/index.html b/docs/session/index.html
index f4b993b..600939f 100644
--- a/docs/session/index.html
+++ b/docs/session/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.session API documentation
-
+
@@ -198,6 +198,19 @@ Methods
The acct_id is a string representing an account within a CSP. The
returned session object is ready to use and loaded with the requested
credentials.
+
+
+Expand source code
+
+def session(self, acct_id):
+ """Returns a session with credentials for the requested account.
+
+ The `acct_id` is a string representing an account within a CSP. The
+ returned session object is ready to use and loaded with the requested
+ credentials.
+ """
+ raise NotImplementedError
+
@@ -237,7 +250,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/src/awsrun/__init__.py b/src/awsrun/__init__.py
index a8d690d..4ae098c 100644
--- a/src/awsrun/__init__.py
+++ b/src/awsrun/__init__.py
@@ -122,4 +122,4 @@
"""
name = "awsrun"
-__version__ = "3.2.1"
+__version__ = "3.2.2"